From d193e249100e6f9de6dda192ad98b9272bd30266 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 10 Jun 2016 10:38:13 +0500 Subject: [PATCH 01/79] Raise error if packet layer unknown --- raw_socket_listener/listener.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 6351cf8..89b6ea6 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -348,6 +348,8 @@ func (t *Listener) readPcap() { data = packet.Data()[14:] } else if linkType == layers.LinkTypeNull || linkType == layers.LinkTypeLoop { data = packet.Data()[4:] + } else { + log.Fatal("Unknown packet layer", packet) } version := uint8(data[0]) >> 4 From f4e2b12d0f2d008943432b1211532fb65c60ed00 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 10 Jun 2016 21:15:08 +0600 Subject: [PATCH 02/79] v0.14.1 (#301) * Fix output-file * Fix tunnel interface --- Makefile | 2 +- plugins.go | 2 +- raw_socket_listener/listener.go | 19 ++++++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index f91108e..7e1df0d 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ profile_test: # Used mainly for debugging, because docker container do not have access to parent machine ports run: - $(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --input-http 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" + $(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --input-http 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor run-2: sudo -E go run $(SOURCE) --input-dummy="" --output-tcp localhost:27001 --verbose --debug diff --git a/plugins.go b/plugins.go index 531ff54..dac9def 100644 --- a/plugins.go +++ b/plugins.go @@ -117,7 +117,7 @@ func InitPlugins() { } for _, options := range Settings.outputFile { - registerPlugin(NewFileOutput, options, Settings.outputFileConfig) + registerPlugin(NewFileOutput, options, &Settings.outputFileConfig) } for _, options := range Settings.inputHTTP { diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 89b6ea6..fd4f2c5 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -325,8 +325,16 @@ func (t *Listener) readPcap() { } t.mu.Unlock() - linkType := handle.LinkType() - source := gopacket.NewPacketSource(handle, linkType) + var decoder gopacket.Decoder + + // Special case for tunnel interface https://github.com/google/gopacket/issues/99 + if handle.LinkType() == 12 { + decoder = layers.LayerTypeIPv4 + } else { + decoder = handle.LinkType() + } + + source := gopacket.NewPacketSource(handle, decoder) source.Lazy = true source.NoCopy = true @@ -343,13 +351,14 @@ func (t *Listener) readPcap() { continue } - if linkType == layers.LinkTypeEthernet { + if decoder == layers.LinkTypeEthernet { // Skip ethernet layer, 14 bytes data = packet.Data()[14:] - } else if linkType == layers.LinkTypeNull || linkType == layers.LinkTypeLoop { + } else if decoder == layers.LinkTypeNull || decoder == layers.LinkTypeLoop { data = packet.Data()[4:] } else { - log.Fatal("Unknown packet layer", packet) + log.Println("Unknown packet layer", packet) + break } version := uint8(data[0]) >> 4 From c4271ffaf48387db77a72fa7ed6175b10484c2bb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 12 Jun 2016 16:19:51 +0500 Subject: [PATCH 03/79] Connection timeout should inherit Timeout --- http_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http_client.go b/http_client.go index 0402351..361ecc6 100644 --- a/http_client.go +++ b/http_client.go @@ -66,7 +66,7 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { config.Timeout = 5 * time.Second } - config.ConnectionTimeout = time.Second + config.ConnectionTimeout = config.Timeout if config.ResponseBufferSize == 0 { config.ResponseBufferSize = 100 * 1024 // 100kb From 95eecb6b1cd97ce85bc3220766072497855f005e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 14 Jun 2016 20:27:52 +0500 Subject: [PATCH 04/79] Output file fixes * Fixed flush/Write race * Fixed names with underscores --- output_file.go | 20 ++++++++++++++++---- output_file_test.go | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/output_file.go b/output_file.go index 5b797d8..fc0965e 100644 --- a/output_file.go +++ b/output_file.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "time" + "sync" ) var dateFileNameFuncs = map[string]func() string{ @@ -33,6 +34,7 @@ type FileOutputConfig struct { // FileOutput output plugin type FileOutput struct { + mu sync.Mutex pathTemplate string currentName string file *os.File @@ -87,7 +89,9 @@ func setFileIndex(name string, idx int) string { withoutExt := strings.TrimSuffix(name, ext) if i := strings.LastIndex(withoutExt, "_"); i != -1 { - withoutExt = withoutExt[:i] + if _, err := strconv.Atoi(withoutExt[i+1:]); err == nil { + withoutExt = withoutExt[:i] + } } return withoutExt + "_" + idxS + ext @@ -120,6 +124,9 @@ func (s sortByFileIndex) Less(i, j int) bool { } func (o *FileOutput) filename() string { + defer o.mu.Unlock() + o.mu.Lock() + path := o.pathTemplate for name, fn := range dateFileNameFuncs { @@ -172,6 +179,7 @@ func (o *FileOutput) Write(data []byte) (n int, err error) { } if o.file == nil || o.currentName != o.file.Name() { + o.mu.Lock() o.Close() o.file, err = os.OpenFile(o.currentName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) @@ -188,6 +196,7 @@ func (o *FileOutput) Write(data []byte) (n int, err error) { } o.queueLength = 0 + o.mu.Unlock() } o.writer.Write(data) @@ -199,16 +208,19 @@ func (o *FileOutput) Write(data []byte) (n int, err error) { } func (o *FileOutput) flush() { + defer o.mu.Unlock() + o.mu.Lock() + if o.file != nil { if strings.HasSuffix(o.currentName, ".gz") { o.writer.(*gzip.Writer).Flush() } else { o.writer.(*bufio.Writer).Flush() } - } - if stat, err := o.file.Stat(); err != nil { - o.chunkSize = int(stat.Size()) + if stat, err := o.file.Stat(); err != nil { + o.chunkSize = int(stat.Size()) + } } } diff --git a/output_file_test.go b/output_file_test.go index ec79351..b086740 100644 --- a/output_file_test.go +++ b/output_file_test.go @@ -166,6 +166,7 @@ func TestSetFileIndex(t *testing.T) { {"/tmp/logs_1", 0, "/tmp/logs_0"}, {"/tmp/logs_0", 10, "/tmp/logs_10"}, {"/tmp/logs_0.gz", 10, "/tmp/logs_10.gz"}, + {"/tmp/logs_underscores.gz", 10, "/tmp/logs_underscores_10.gz"}, } for _, c := range tests { From d0e1858eb3323bc4b43c058df3191c0391d6c240 Mon Sep 17 00:00:00 2001 From: Marc Lallaouret Date: Thu, 16 Jun 2016 18:33:07 +0200 Subject: [PATCH 05/79] Fix `Close` implementation of plugins (#305) Indeed they do not declare error as return type and so, were not called at the end of the program --- input_file.go | 3 ++- input_raw.go | 3 ++- output_file.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/input_file.go b/input_file.go index b562a99..ba3f542 100644 --- a/input_file.go +++ b/input_file.go @@ -189,6 +189,7 @@ func (i *FileInput) emit() { log.Printf("FileInput: end of file '%s'\n", i.path) } -func (i *FileInput) Close() { +func (i *FileInput) Close() error { i.currentFile.Close() + return nil } diff --git a/input_raw.go b/input_raw.go index c67cd8c..7b92096 100644 --- a/input_raw.go +++ b/input_raw.go @@ -97,7 +97,8 @@ func (i *RAWInput) String() string { return "Intercepting traffic from: " + i.address } -func (i *RAWInput) Close() { +func (i *RAWInput) Close() error { i.listener.Close() close(i.quit) + return nil } diff --git a/output_file.go b/output_file.go index fc0965e..4c863fc 100644 --- a/output_file.go +++ b/output_file.go @@ -228,7 +228,7 @@ func (o *FileOutput) String() string { return "File output: " + o.file.Name() } -func (o *FileOutput) Close() { +func (o *FileOutput) Close() error { if o.file != nil { if strings.HasSuffix(o.currentName, ".gz") { o.writer.(*gzip.Writer).Close() @@ -237,4 +237,5 @@ func (o *FileOutput) Close() { } o.file.Close() } + return nil } From 2c49556b2eedfe959202659ae56f0fdf04207f42 Mon Sep 17 00:00:00 2001 From: Marc Lallaouret Date: Thu, 16 Jun 2016 18:33:55 +0200 Subject: [PATCH 06/79] Clean current output file name (#304) --- output_file.go | 4 ++-- output_file_test.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/output_file.go b/output_file.go index 4c863fc..044068d 100644 --- a/output_file.go +++ b/output_file.go @@ -11,8 +11,8 @@ import ( "sort" "strconv" "strings" - "time" "sync" + "time" ) var dateFileNameFuncs = map[string]func() string{ @@ -170,7 +170,7 @@ func (o *FileOutput) filename() string { } func (o *FileOutput) updateName() { - o.currentName = o.filename() + o.currentName = filepath.Clean(o.filename()) } func (o *FileOutput) Write(data []byte) (n int, err error) { diff --git a/output_file_test.go b/output_file_test.go index b086740..04601f8 100644 --- a/output_file_test.go +++ b/output_file_test.go @@ -53,6 +53,17 @@ func TestFileOutput(t *testing.T) { close(quit) } +func TestFileOutputWithNameCleaning(t *testing.T) { + output := &FileOutput{pathTemplate: "./test_requests.gor", config: &FileOutputConfig{flushInterval: time.Minute, append: false}} + expectedFileName := "test_requests_0.gor" + output.updateName() + + if expectedFileName != output.currentName { + t.Errorf("Expected path %s but got %s", expectedFileName, output.currentName) + } + +} + func TestFileOutputPathTemplate(t *testing.T) { output := &FileOutput{pathTemplate: "/tmp/log-%Y-%m-%d-%S", config: &FileOutputConfig{flushInterval: time.Minute, append: true}} now := time.Now() From 9e7351d28c19914bce9cf2f465a8e97c6556e3e0 Mon Sep 17 00:00:00 2001 From: Yohan Legat Date: Fri, 10 Jun 2016 15:24:21 +0200 Subject: [PATCH 07/79] Resolve buger/gor#300 : respect timestamps when replaying requests When Gor deals with multiple input files, it sorts them and replay their requests file after file. This behavior is not convenient and should be changed. We should respect timestamps order when replaying requests. --- input_file.go | 215 ++++++++++++++++++++++++++------------------- input_file_test.go | 102 ++++++++++++++++++--- 2 files changed, 211 insertions(+), 106 deletions(-) diff --git a/input_file.go b/input_file.go index ba3f542..8170179 100644 --- a/input_file.go +++ b/input_file.go @@ -9,31 +9,40 @@ import ( "log" "os" "path/filepath" - "sort" - "strconv" "strings" + "strconv" "time" ) // FileInput can read requests generated by FileOutput type FileInput struct { - data chan []byte - path string - currentFile *os.File - currentReader *bufio.Reader - speedFactor float64 - loop bool + data chan []byte + exit chan bool + path string + fileInputReaders []*fileInputReader + speedFactor float64 + loop bool +} + + +type fileInputReader struct { + reader *bufio.Reader + meta [][]byte + data []byte + file *os.File + timestamp int64 } // NewFileInput constructor for FileInput. Accepts file path as argument. func NewFileInput(path string, loop bool) (i *FileInput) { i = new(FileInput) i.data = make(chan []byte) + i.exit = make(chan bool) i.path = path i.speedFactor = 1 i.loop = loop - if err := i.updateFile(); err != nil { + if err := i.init(); err != nil { return } @@ -48,9 +57,7 @@ func (_ *NextFileNotFound) Error() string { return "There is no new files" } -// path can be a pattern -// It sort paths lexicographically and tries to choose next one -func (i *FileInput) updateFile() (err error) { +func (i *FileInput) init() (err error) { var matches []string if matches, err = filepath.Glob(i.path); err != nil { @@ -63,40 +70,24 @@ func (i *FileInput) updateFile() (err error) { return errors.New("No matching files") } - sort.Sort(sortByFileIndex(matches)) + i.fileInputReaders = make([]*fileInputReader, len(matches)) - // Just pick first file, if there is many, and we are just started - if i.currentFile == nil { - if i.currentFile, err = os.Open(matches[0]); err != nil { - log.Println("Can't read file ", matches[0], err) - return - } - } else { - found := false - for idx, p := range matches { - if p == i.currentFile.Name() && idx != len(matches)-1 { - if i.currentFile, err = os.Open(matches[idx+1]); err != nil { - log.Println("Can't read file ", matches[idx+1], err) - return - } - - found = true + for idx, p := range matches { + file, _ := os.Open(p) + fileInputReader := &fileInputReader{} + fileInputReader.file = file + if strings.HasSuffix(p, ".gz") { + gzReader, err := gzip.NewReader(file) + if err != nil { + log.Fatal(err) } + fileInputReader.reader = bufio.NewReader(gzReader) + } else { + fileInputReader.reader = bufio.NewReader(file) } - if !found { - return new(NextFileNotFound) - } - } - - if strings.HasSuffix(i.currentFile.Name(), ".gz") { - gzReader, err := gzip.NewReader(i.currentFile) - if err != nil { - log.Fatal(err) - } - i.currentReader = bufio.NewReader(gzReader) - } else { - i.currentReader = bufio.NewReader(i.currentFile) + fileInputReader.readNextInput() + i.fileInputReaders[idx] = fileInputReader } return nil @@ -113,83 +104,123 @@ func (i *FileInput) String() string { return "File input: " + i.path } -func (i *FileInput) emit() { - var lastTime int64 +func (f *fileInputReader) readNextInput() { + nextInput := f.nextInput() + f.parseNextInput(nextInput) +} +func (f *fileInputReader) parseNextInput(input []byte) { + if (input != nil) { + f.meta = payloadMeta(input) + f.timestamp, _ = strconv.ParseInt(string(f.meta[2]), 10, 64) + f.data = input + } +} + +func (f *fileInputReader) nextInput() []byte { payloadSeparatorAsBytes := []byte(payloadSeparator) - var buffer bytes.Buffer - if i.currentReader == nil { - return - } - for { - line, err := i.currentReader.ReadBytes('\n') + line, err := f.reader.ReadBytes('\n') if err != nil { if err != io.EOF { log.Fatal(err) } - // If our path pattern match multiple files, try to find them if err == io.EOF { - if e := i.updateFile(); e != nil { - if _, ok := e.(*NextFileNotFound); ok && i.loop { - // Start from the first file - i.Close() - i.currentFile = nil - i.currentReader = nil - lastTime = 0 - i.updateFile() - - continue - } else { - break - } - } - - continue + f.file.Close() + f.file = nil + return nil } } if bytes.Equal(payloadSeparatorAsBytes[1:], line) { asBytes := buffer.Bytes() - buffer.Reset() - - meta := payloadMeta(asBytes) - - if len(meta) > 2 && meta[0][0] == RequestPayload { - ts, _ := strconv.ParseInt(string(meta[2]), 10, 64) - - if lastTime != 0 { - timeDiff := ts - lastTime - - if i.speedFactor != 1 { - timeDiff = int64(float64(timeDiff) / i.speedFactor) - } - - time.Sleep(time.Duration(timeDiff)) - } - - lastTime = ts - } // Bytes() returns only pointer, so to remove data-race copy the data to an array - newBuf := make([]byte, len(asBytes)-1) + newBuf := make([]byte, len(asBytes) - 1) copy(newBuf, asBytes) - - i.data <- newBuf - } else { - buffer.Write(line) + return newBuf } + buffer.Write(line) + } +} + +func (i *FileInput) nextInputReader() *fileInputReader { + var nextFileInputReader *fileInputReader + for _, fileInputReader := range i.fileInputReaders { + if fileInputReader.file == nil { + continue + } + + if fileInputReader.meta[0][0] == ResponsePayload { + return fileInputReader + } + + if nextFileInputReader == nil || nextFileInputReader.timestamp > fileInputReader.timestamp { + nextFileInputReader = fileInputReader + continue + } + } + + return nextFileInputReader; +} + +func (i *FileInput) emit() { + var lastTime int64 = -1 + + for { + fileInputReader := i.nextInputReader() + + if fileInputReader == nil { + if i.loop { + i.init() + lastTime = -1 + continue + } else { + break; + } + } + + if fileInputReader.meta[0][0] == RequestPayload { + lastTime = i.simulateRequestDelay(fileInputReader, lastTime) + } + + select { + case <-i.exit: + for _, fileInputReader := range i.fileInputReaders { + if fileInputReader.file != nil { + fileInputReader.file.Close() + } + } + break + case i.data <- fileInputReader.data: + fileInputReader.readNextInput() + } } log.Printf("FileInput: end of file '%s'\n", i.path) } +func (i*FileInput) simulateRequestDelay(fileInputReader *fileInputReader, lastTime int64) int64 { + if lastTime != -1 { + timeDiff := fileInputReader.timestamp - lastTime + + if i.speedFactor != 1 { + timeDiff = int64(float64(timeDiff) / i.speedFactor) + } + + time.Sleep(time.Duration(timeDiff)) + } + + return fileInputReader.timestamp +} + func (i *FileInput) Close() error { - i.currentFile.Close() + i.exit <- true return nil } + diff --git a/input_file_test.go b/input_file_test.go index e5857cf..61c18cd 100644 --- a/input_file_test.go +++ b/input_file_test.go @@ -97,35 +97,109 @@ func TestInputFileWithGETAndPOST(t *testing.T) { } -func TestInputFileMultipleFiles(t *testing.T) { +func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) { rnd := rand.Int63() file1, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_0", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) file1.Write([]byte("1 1 1\ntest1")) file1.Write([]byte(payloadSeparator)) - file1.Write([]byte("1 1 2\ntest2")) + file1.Write([]byte("1 1 3\ntest2")) file1.Write([]byte(payloadSeparator)) file1.Close() file2, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_1", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) - file2.Write([]byte("1 1 3\ntest3")) + file2.Write([]byte("1 1 2\ntest3")) + file2.Write([]byte(payloadSeparator)) + file2.Write([]byte("1 1 4\ntest4")) file2.Write([]byte(payloadSeparator)) file2.Close() input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false) buf := make([]byte, 1000) - n, _ := input.Read(buf) - if buf[10] != '1' { - t.Error("Shound emit requests in right order", string(buf[:n])) - } - input.Read(buf) - if buf[10] != '2' { - t.Error("Shound emit requests in right order", string(buf[:n])) + + for i := '1'; i <= '4'; i++ { + n, _ := input.Read(buf) + if buf[4] != byte(i) { + t.Error("Should emit requests in right order", string(buf[:n])) + } } - input.Read(buf) - if buf[10] != '3' { - t.Error("Shound emit requests from second file", string(buf[:n])) + os.Remove(file1.Name()) + os.Remove(file2.Name()) +} + +func TestInputFileRequestsWithLatency(t *testing.T) { + rnd := rand.Int63() + + file, _ := os.OpenFile(fmt.Sprintf("/tmp/%d", rnd), os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0660) + defer file.Close() + + file.Write([]byte("1 1 100000000\nrequest1")) + file.Write([]byte(payloadSeparator)) + file.Write([]byte("1 2 150000000\nrequest2")) + file.Write([]byte(payloadSeparator)) + file.Write([]byte("1 3 250000000\nrequest3")) + file.Write([]byte(payloadSeparator)) + + input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false) + buf := make([]byte, 1000) + + start := time.Now().UnixNano() + for i := 0; i < 3; i++ { + input.Read(buf) + } + end := time.Now().UnixNano() + + var expectedLatency int64 = 250000000 - 100000000 + realLatency := end - start + if realLatency < expectedLatency { + t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency) + } + + if realLatency > expectedLatency + 10000000 { + t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency) + + } +} + +func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) { + rnd := rand.Int63() + + file1, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_0", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) + file1.Write([]byte("1 1 1\nrequest1")) + file1.Write([]byte(payloadSeparator)) + file1.Write([]byte("2 1 1\nresponse1")) + file1.Write([]byte(payloadSeparator)) + file1.Write([]byte("1 2 3\nrequest2")) + file1.Write([]byte(payloadSeparator)) + file1.Write([]byte("2 2 3\nresponse2")) + file1.Write([]byte(payloadSeparator)) + file1.Close() + + file2, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_1", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) + file2.Write([]byte("1 3 2\nrequest3")) + file2.Write([]byte(payloadSeparator)) + file2.Write([]byte("2 3 2\nresponse3")) + file2.Write([]byte(payloadSeparator)) + file2.Write([]byte("1 4 4\nrequest4")) + file2.Write([]byte(payloadSeparator)) + file2.Write([]byte("2 4 4\nresponse4")) + file2.Write([]byte(payloadSeparator)) + file2.Close() + + input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false) + buf := make([]byte, 1000) + + for i := '1'; i <= '4'; i++ { + n, _ := input.Read(buf) + if buf[0] != '1' && buf[4] != byte(i) { + t.Error("Shound emit requests in right order", string(buf[:n])) + } + + n, _ = input.Read(buf) + if buf[0] != '2' && buf[4] != byte(i) { + t.Error("Shound emit responses in right order", string(buf[:n])) + } } os.Remove(file1.Name()) @@ -149,8 +223,8 @@ func TestInputFileLoop(t *testing.T) { for i := 0; i < 1000; i++ { input.Read(buf) } - input.Close() + input.Close() os.Remove(file.Name()) } From 51860e130cb25902e332fc6a992d04f96ebad312 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 20 Jun 2016 19:32:13 +0300 Subject: [PATCH 08/79] Add support for known network layers Fix #310 Close #311 --- raw_socket_listener/listener.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index fd4f2c5..59bbcf9 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -351,16 +351,30 @@ func (t *Listener) readPcap() { continue } - if decoder == layers.LinkTypeEthernet { - // Skip ethernet layer, 14 bytes - data = packet.Data()[14:] - } else if decoder == layers.LinkTypeNull || decoder == layers.LinkTypeLoop { - data = packet.Data()[4:] - } else { - log.Println("Unknown packet layer", packet) - break + // We should remove network layer before parsing TCP/IP data + var of int + switch decoder { + case layers.LinkTypeEthernet: + of = 14 + case layers.LinkTypePPP: + of = 1 + case layers.LinkTypeFDDI: + of = 13 + case layers.LinkTypeNull: + of = 4 + case layers.LinkTypeLoop: + of = 4 + case layers.LinkTypeRaw: + of = 0 + case layers.LinkTypeLinuxSLL: + of = 16 + default: + log.Println("Unknown packet layer", packet) + break } + data = packet.Data()[of:] + version := uint8(data[0]) >> 4 if version == 4 { From e50ff77ca63460f2d5ec2a1f83af238541e6c9d2 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 20 Jun 2016 19:57:10 +0300 Subject: [PATCH 09/79] Update echo.sh --- examples/middleware/echo.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/middleware/echo.sh b/examples/middleware/echo.sh index ec946ee..f6f10e0 100755 --- a/examples/middleware/echo.sh +++ b/examples/middleware/echo.sh @@ -2,6 +2,10 @@ # # `xxd` utility included into vim-common package # It allow hex decoding/encoding +# +# This example may broke if you request contains `null` string, you may consider using pipes instead. +# See: https://github.com/buger/gor/issues/309 +# function log { # Logging to stderr, because stdout/stdin used for data transfer From da09b14c8b3a292618b728ff4d2a2ac854442722 Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Mon, 20 Jun 2016 11:02:12 -0400 Subject: [PATCH 10/79] DRY 100-continue tests --- raw_socket_listener/listener_test.go | 97 +++++++++------------------- 1 file changed, 31 insertions(+), 66 deletions(-) diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index f5d7d49..8d2b610 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -108,9 +108,7 @@ func TestRawListenerResponse(t *testing.T) { } } -func TestRawListener100Continue(t *testing.T) { - var req, resp *TCPMessage - +func TestShort100Continue(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() @@ -124,12 +122,36 @@ func TestRawListener100Continue(t *testing.T) { // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n")) - listener.packetsChan <- reqPacket1.Dump() - listener.packetsChan <- reqPacket2.Dump() - listener.packetsChan <- reqPacket3.Dump() + result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") - listener.packetsChan <- respPacket1.Dump() - listener.packetsChan <- respPacket2.Dump() + testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2) +} + +// Response comes before Request +func Test100ContinueWrongOrder(t *testing.T) { + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + defer listener.Close() + + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n")) + // Packet with data have different Seq + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) + + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) + + // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n")) + + result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") + + testRawListener100Continue(t, listener, result, respPacket1, respPacket2, reqPacket1, reqPacket2, reqPacket3) +} + +func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) { + var req, resp *TCPMessage + for _, p := range packets { + listener.packetsChan <- p.Dump() + } select { case req = <-listener.messagesChan: @@ -139,64 +161,7 @@ func TestRawListener100Continue(t *testing.T) { return } - if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")) { - t.Error("Should receive full message", string(req.Bytes())) - } - - if !req.IsIncoming { - t.Error("Should be request") - } - - select { - case resp = <-listener.messagesChan: - break - case <-time.After(21 * time.Millisecond): - t.Error("Should return response after expire time") - return - } - - if resp.IsIncoming { - t.Error("Should be response") - } - - if !bytes.Equal(resp.UUID(), req.UUID()) { - t.Error("Resp and Req UUID should be equal") - } -} - -// Response comes before Request -func TestRawListener100ContinueWrongOrder(t *testing.T) { - var req, resp *TCPMessage - - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) - defer listener.Close() - - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n")) - // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) - - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) - - // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n")) - - listener.packetsChan <- respPacket1.Dump() - listener.packetsChan <- respPacket2.Dump() - - listener.packetsChan <- reqPacket1.Dump() - listener.packetsChan <- reqPacket2.Dump() - listener.packetsChan <- reqPacket3.Dump() - - select { - case req = <-listener.messagesChan: - break - case <-time.After(11 * time.Millisecond): - t.Error("Should return response after expire time") - return - } - - if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")) { + if !bytes.Equal(req.Bytes(), result) { t.Error("Should receive full message", string(req.Bytes())) } From b0fff3c4029d9293cdfa33e9e39cefb018059d44 Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Mon, 20 Jun 2016 11:59:00 -0400 Subject: [PATCH 11/79] Alternative 100 continue test --- raw_socket_listener/listener_test.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index 8d2b610..3f3b250 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -147,6 +147,27 @@ func Test100ContinueWrongOrder(t *testing.T) { testRawListener100Continue(t, listener, result, respPacket1, respPacket2, reqPacket1, reqPacket2, reqPacket3) } +func TestAlt100ContinueHeaderOrder(t *testing.T) { + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + defer listener.Close() + + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n")) + // Packet with data have different Seq + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) + + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) + + // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n")) + + result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") + + testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2) +} + + + func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) { var req, resp *TCPMessage for _, p := range packets { @@ -157,7 +178,7 @@ func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, case req = <-listener.messagesChan: break case <-time.After(11 * time.Millisecond): - t.Error("Should return request after expire time") + t.Error("Should return response after expire time") return } From 5d2cc68fffac53b89b141ed6fbf1b7e992071d0c Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Mon, 20 Jun 2016 16:14:54 -0400 Subject: [PATCH 12/79] Add proto.DelHeader Update proto.go to handle headers with whitespace after --- proto/proto.go | 50 ++++++++++++++++++++++++++++++++++----------- proto/proto_test.go | 28 ++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/proto/proto.go b/proto/proto.go index 0e74cc8..4e15a5a 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -123,7 +123,7 @@ func headerIndex(payload []byte, name []byte) int { // header return value and positions of header/value start/end. // If not found, value will be blank, and headerStart will be -1 // Do not support multi-line headers. -func header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) { +func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) { headerStart = headerIndex(payload, name) if headerStart == -1 { @@ -131,24 +131,37 @@ func header(payload []byte, name []byte) (value []byte, headerStart, valueStart, } valueStart = headerStart + len(name) + 1 // Skip ":" after header name - if payload[valueStart] == ' ' { // Ignore empty space after ':' - valueStart++ - } - headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n') - if payload[headerEnd-1] == '\r' { - headerEnd -= 1 + for valueStart < headerEnd { // Ignore empty space after ':' + if payload[valueStart] == ' ' { + valueStart++ + } else { + break + } } - value = payload[valueStart:headerEnd] + valueEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n') + + if payload[headerEnd-1] == '\r' { + valueEnd -= 1 + } + + for valueStart < valueEnd { // ignore empty space at end of header value + if payload[valueEnd-1] == ' ' { + valueEnd -= 1 + } else { + break + } + } + value = payload[valueStart:valueEnd] return } // Header returns header value, if header not found, value will be blank func Header(payload, name []byte) []byte { - val, _, _, _ := header(payload, name) + val, _, _, _, _ := header(payload, name) return val } @@ -156,11 +169,11 @@ func Header(payload, name []byte) []byte { // SetHeader sets header value. If header not found it creates new one. // Returns modified request payload func SetHeader(payload, name, value []byte) []byte { - _, hs, vs, he := header(payload, name) + _, hs, _, vs, ve := header(payload, name) if hs != -1 { - // If header found we just repace its value - return byteutils.Replace(payload, vs, he, value) + // If header found we just replace its value + return byteutils.Replace(payload, vs, ve, value) } return AddHeader(payload, name, value) @@ -180,6 +193,19 @@ func AddHeader(payload, name, value []byte) []byte { return byteutils.Insert(payload, mimeStart, header) } +// DelHeader takes http payload and removes header name from headers section +// Returns modified request payload +func DelHeader(payload, name[]byte) []byte { + _, hs, he, _, _ := header(payload, name) + if hs != -1 { + newHeader := make([]byte, len(payload) - (he - hs) - 1) + copy(newHeader[:hs], payload[:hs]) + copy(newHeader[hs:], payload[he + 1:]) + return newHeader + } + return payload +} + // Body returns request/response body func Body(payload []byte) []byte { // 4 -> len(EMPTY_LINE) diff --git a/proto/proto_test.go b/proto/proto_test.go index a8d9179..2510237 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -16,6 +16,13 @@ func TestHeader(t *testing.T) { t.Error("Should find header value") } + // Value with space at end + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7 \r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { + t.Error("Should find header value without space after 7") + } + // Value without space at start payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2") @@ -38,7 +45,7 @@ func TestHeader(t *testing.T) { } // Header not found - if _, headerStart, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 { + if _, headerStart, _, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 { t.Error("Should not found header") } @@ -97,6 +104,25 @@ func TestSetHeader(t *testing.T) { } } +func TestDelHeader(t *testing.T) { + var payload, payloadAfter []byte + + payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = DelHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) { + t.Error("Should delete header if found", string(payload), string(payloadAfter)) + } + + //Whitespace at end of User-Agent + payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor \r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = DelHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) { + t.Error("Should delete header if found", string(payload), string(payloadAfter)) + } +} + func TestPath(t *testing.T) { var path, payload []byte From 0089892b26c21d5e0ccdf7d8c4ef89ad2c9f48bd Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Mon, 20 Jun 2016 16:35:20 -0400 Subject: [PATCH 13/79] update 100-continue logic to support different header placement --- proto/proto.go | 4 +-- raw_socket_listener/listener.go | 52 ++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/proto/proto.go b/proto/proto.go index 4e15a5a..5376473 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -144,12 +144,12 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n') if payload[headerEnd-1] == '\r' { - valueEnd -= 1 + valueEnd-- } for valueStart < valueEnd { // ignore empty space at end of header value if payload[valueEnd-1] == ' ' { - valueEnd -= 1 + valueEnd-- } else { break } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 59bbcf9..e1b0a29 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -28,6 +28,7 @@ import ( "strings" "sync" "time" + "github.com/buger/gor/proto" ) var _ = fmt.Println @@ -515,8 +516,10 @@ func (t *Listener) isValidPacket(buf []byte) bool { return false } -var bExpect100ContinueCheck = []byte("Expect: 100-continue") +var bExpectHeader = []byte("Expect:") +var bExpect100Value = []byte("100-continue") var bPOST = []byte("POST") +var bCRLFx2 = []byte("\r\n\r\n") // Trying to add packet to existing message or creating new message // @@ -590,33 +593,36 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { // Handling Expect: 100-continue requests if len(packet.Data) > 4 && bytes.Equal(packet.Data[0:4], bPOST) { - // reading last 20 bytes (not counting CRLF): last header value (if no body presented) - if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { - seq := packet.Seq + uint32(len(packet.Data)) - t.seqWithData[seq] = packet.Ack - message.DataSeq = seq + // reading last 8 bytes for double CRLF + if bytes.Equal(packet.Data[len(packet.Data)-4:], bCRLFx2) { + // look for an expect:100-continue header + if bytes.Equal(bExpect100Value, proto.Header(packet.Data, bExpectHeader)) { + seq := packet.Seq + uint32(len(packet.Data)) + t.seqWithData[seq] = packet.Ack + message.DataSeq = seq - // In case if sequence packet came first - for _, m := range t.messages { - if m.Seq == seq { - t.deleteMessage(m) - if m.AssocMessage != nil { - message.AssocMessage = m.AssocMessage - } - // log.Println("2: Adding ack alias:", m.Ack, packet.Ack) - t.ackAliases[m.Ack] = packet.Ack + // In case if sequence packet came first + for _, m := range t.messages { + if m.Seq == seq { + t.deleteMessage(m) + if m.AssocMessage != nil { + message.AssocMessage = m.AssocMessage + } + // log.Println("2: Adding ack alias:", m.Ack, packet.Ack) + t.ackAliases[m.Ack] = packet.Ack - for _, pkt := range m.packets { - pkt.UpdateAck(packet.Ack) - message.AddPacket(pkt) + for _, pkt := range m.packets { + pkt.UpdateAck(packet.Ack) + message.AddPacket(pkt) + } } } + + // Removing `Expect: 100-continue` header + packet.Data = proto.DelHeader(packet.Data, bExpectHeader) + + // log.Println(string(packet.Data)) } - - // Removing `Expect: 100-continue` header - packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...) - - // log.Println(string(packet.Data)) } } From aea32d93952c5ae4b3a3c1c36b4ee0252347c15a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 21 Jun 2016 18:05:32 +0300 Subject: [PATCH 14/79] Rename DelHeader to DeleteHeader --- proto/proto.go | 5 +++-- proto/proto_test.go | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/proto/proto.go b/proto/proto.go index 5376473..40e59b1 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -147,7 +147,8 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueEnd-- } - for valueStart < valueEnd { // ignore empty space at end of header value + // ignore empty space at end of header value + for valueStart < valueEnd { if payload[valueEnd-1] == ' ' { valueEnd-- } else { @@ -195,7 +196,7 @@ func AddHeader(payload, name, value []byte) []byte { // DelHeader takes http payload and removes header name from headers section // Returns modified request payload -func DelHeader(payload, name[]byte) []byte { +func DeleteHeader(payload, name[]byte) []byte { _, hs, he, _, _ := header(payload, name) if hs != -1 { newHeader := make([]byte, len(payload) - (he - hs) - 1) diff --git a/proto/proto_test.go b/proto/proto_test.go index 2510237..897bc4f 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -104,13 +104,13 @@ func TestSetHeader(t *testing.T) { } } -func TestDelHeader(t *testing.T) { +func TestDeleteHeader(t *testing.T) { var payload, payloadAfter []byte payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = DelHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) { + if payload = DeleteHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) { t.Error("Should delete header if found", string(payload), string(payloadAfter)) } @@ -118,7 +118,7 @@ func TestDelHeader(t *testing.T) { payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor \r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = DelHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) { + if payload = DeleteHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) { t.Error("Should delete header if found", string(payload), string(payloadAfter)) } } From 48fdc02d2f6d84e0d9cd3d1b459313f67735fe31 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 21 Jun 2016 18:32:07 +0300 Subject: [PATCH 15/79] Fix tests + refactoring --- raw_socket_listener/listener.go | 51 +++++++++++------------------- raw_socket_listener/tcp_message.go | 29 +++++++++++++++++ 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index e1b0a29..3cd5e70 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -516,11 +516,6 @@ func (t *Listener) isValidPacket(buf []byte) bool { return false } -var bExpectHeader = []byte("Expect:") -var bExpect100Value = []byte("100-continue") -var bPOST = []byte("POST") -var bCRLFx2 = []byte("\r\n\r\n") - // Trying to add packet to existing message or creating new message // // For TCP message unique id is Acknowledgment number (see tcp_packet.go) @@ -592,38 +587,30 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { message.AddPacket(packet) // Handling Expect: 100-continue requests - if len(packet.Data) > 4 && bytes.Equal(packet.Data[0:4], bPOST) { - // reading last 8 bytes for double CRLF - if bytes.Equal(packet.Data[len(packet.Data)-4:], bCRLFx2) { - // look for an expect:100-continue header - if bytes.Equal(bExpect100Value, proto.Header(packet.Data, bExpectHeader)) { - seq := packet.Seq + uint32(len(packet.Data)) - t.seqWithData[seq] = packet.Ack - message.DataSeq = seq + if message.Is100Continue() { + seq := packet.Seq + uint32(len(packet.Data)) + t.seqWithData[seq] = packet.Ack + message.DataSeq = seq - // In case if sequence packet came first - for _, m := range t.messages { - if m.Seq == seq { - t.deleteMessage(m) - if m.AssocMessage != nil { - message.AssocMessage = m.AssocMessage - } - // log.Println("2: Adding ack alias:", m.Ack, packet.Ack) - t.ackAliases[m.Ack] = packet.Ack - - for _, pkt := range m.packets { - pkt.UpdateAck(packet.Ack) - message.AddPacket(pkt) - } - } + // In case if sequence packet came first + for _, m := range t.messages { + if m.Seq == seq { + t.deleteMessage(m) + if m.AssocMessage != nil { + message.AssocMessage = m.AssocMessage } + // log.Println("2: Adding ack alias:", m.Ack, packet.Ack) + t.ackAliases[m.Ack] = packet.Ack - // Removing `Expect: 100-continue` header - packet.Data = proto.DelHeader(packet.Data, bExpectHeader) - - // log.Println(string(packet.Data)) + for _, pkt := range m.packets { + pkt.UpdateAck(packet.Ack) + message.AddPacket(pkt) + } } } + + // Removing `Expect: 100-continue` header + packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader) } // log.Println("Received message:", string(message.Bytes()), message.ID(), t.messages) diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index f533ad6..5045f4d 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -227,6 +227,35 @@ func (t *TCPMessage) IsFinished() bool { return false } +var bExpectHeader = []byte("Expect:") +var bExpect100Value = []byte("100-continue") +var bPOST = []byte("POST") +var bCRLFx2 = []byte("\r\n\r\n") + +func (t *TCPMessage) Is100Continue() bool { + d := t.packets[0].Data + + if len(d) < 25 { + return false + } + + if !bytes.Equal(d[0:4], bPOST) { + return false + } + + // reading last 4 bytes for double CRLF + if !bytes.Equal(d[len(d)-4:], bCRLFx2) { + return false + } + + // look for an expect:100-continue header + if !bytes.Equal(bExpect100Value, proto.Header(d, bExpectHeader)) { + return false + } + + return true +} + func (t *TCPMessage) UUID() []byte { var key []byte From c756a11333905d39706180a6dd8245b9ee5f77d2 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 27 Jun 2016 16:00:57 +0300 Subject: [PATCH 16/79] Pass only valid HTTP responses (#317) --- proto/proto.go | 14 +- raw_socket_listener/listener.go | 83 +++--- raw_socket_listener/listener_test.go | 26 +- raw_socket_listener/tcp_message.go | 336 +++++++++++++++++------- raw_socket_listener/tcp_message_test.go | 172 +++++++----- 5 files changed, 406 insertions(+), 225 deletions(-) diff --git a/proto/proto.go b/proto/proto.go index 40e59b1..186926f 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -133,7 +133,7 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart = headerStart + len(name) + 1 // Skip ":" after header name headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n') - for valueStart < headerEnd { // Ignore empty space after ':' + for valueStart < headerEnd { // Ignore empty space after ':' if payload[valueStart] == ' ' { valueStart++ } else { @@ -148,7 +148,7 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, } // ignore empty space at end of header value - for valueStart < valueEnd { + for valueStart < valueEnd { if payload[valueEnd-1] == ' ' { valueEnd-- } else { @@ -170,7 +170,7 @@ func Header(payload, name []byte) []byte { // SetHeader sets header value. If header not found it creates new one. // Returns modified request payload func SetHeader(payload, name, value []byte) []byte { - _, hs, _, vs, ve := header(payload, name) + _, hs, _, vs, ve := header(payload, name) if hs != -1 { // If header found we just replace its value @@ -196,12 +196,12 @@ func AddHeader(payload, name, value []byte) []byte { // DelHeader takes http payload and removes header name from headers section // Returns modified request payload -func DeleteHeader(payload, name[]byte) []byte { +func DeleteHeader(payload, name []byte) []byte { _, hs, he, _, _ := header(payload, name) if hs != -1 { - newHeader := make([]byte, len(payload) - (he - hs) - 1) + newHeader := make([]byte, len(payload)-(he-hs)-1) copy(newHeader[:hs], payload[:hs]) - copy(newHeader[hs:], payload[he + 1:]) + copy(newHeader[hs:], payload[he+1:]) return newHeader } return payload @@ -322,7 +322,7 @@ func Status(payload []byte) []byte { } var httpMethods []string = []string{ - "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", /* custom methods */"BAN", "PURG", + "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN" /* custom methods */, "BAN", "PURG", } func IsHTTPPayload(payload []byte) bool { diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 3cd5e70..88066d2 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -16,6 +16,7 @@ import ( "bytes" "encoding/binary" "fmt" + "github.com/buger/gor/proto" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/google/gopacket/pcap" @@ -28,7 +29,6 @@ import ( "strings" "sync" "time" - "github.com/buger/gor/proto" ) var _ = fmt.Println @@ -173,7 +173,17 @@ func (t *Listener) dispatchMessage(message *TCPMessage) { t.deleteMessage(message) - // log.Println("Dispatching, message", message.Start.UnixNano(), message.Seq, message.Ack, string(message.Bytes())) + if message.methodType == httpMethodNotFound { + return + } + + if !message.complete { + if !message.IsIncoming { + delete(t.respAliases, message.Ack) + delete(t.respWithoutReq, message.Ack) + } + return + } if message.IsIncoming { // If there were response before request @@ -183,10 +193,10 @@ func (t *Listener) dispatchMessage(message *TCPMessage) { if resp, rok := t.messages[respID]; rok { // if resp.AssocMessage == nil { // log.Println("FOUND RESPONSE") - resp.AssocMessage = message - message.AssocMessage = resp + resp.setAssocMessage(message) + message.setAssocMessage(resp) - if resp.IsFinished() { + if resp.complete { defer t.dispatchMessage(resp) } // } @@ -194,14 +204,14 @@ func (t *Listener) dispatchMessage(message *TCPMessage) { } if resp, ok := t.messages[message.ResponseID]; ok { - resp.AssocMessage = message + resp.setAssocMessage(message) } } } else { if message.AssocMessage == nil { if responseRequest, ok := t.respAliases[message.Ack]; ok { - message.AssocMessage = responseRequest - responseRequest.AssocMessage = message + message.setAssocMessage(responseRequest) + responseRequest.setAssocMessage(message) } } @@ -303,7 +313,7 @@ func (t *Listener) readPcap() { for i, addr := range device.Addresses { bpfDstHost += "dst host " + addr.IP.String() bpfSrcHost += "src host " + addr.IP.String() - if i != len(device.Addresses) - 1 { + if i != len(device.Addresses)-1 { bpfDstHost += " or " bpfSrcHost += " or " } @@ -330,9 +340,9 @@ func (t *Listener) readPcap() { // Special case for tunnel interface https://github.com/google/gopacket/issues/99 if handle.LinkType() == 12 { - decoder = layers.LayerTypeIPv4 + decoder = layers.LayerTypeIPv4 } else { - decoder = handle.LinkType() + decoder = handle.LinkType() } source := gopacket.NewPacketSource(handle, decoder) @@ -355,23 +365,23 @@ func (t *Listener) readPcap() { // We should remove network layer before parsing TCP/IP data var of int switch decoder { - case layers.LinkTypeEthernet: - of = 14 - case layers.LinkTypePPP: - of = 1 - case layers.LinkTypeFDDI: - of = 13 - case layers.LinkTypeNull: - of = 4 - case layers.LinkTypeLoop: - of = 4 - case layers.LinkTypeRaw: - of = 0 - case layers.LinkTypeLinuxSLL: - of = 16 - default: - log.Println("Unknown packet layer", packet) - break + case layers.LinkTypeEthernet: + of = 14 + case layers.LinkTypePPP: + of = 1 + case layers.LinkTypeFDDI: + of = 13 + case layers.LinkTypeNull: + of = 4 + case layers.LinkTypeLoop: + of = 4 + case layers.LinkTypeRaw: + of = 0 + case layers.LinkTypeLinuxSLL: + of = 16 + default: + log.Println("Unknown packet layer", packet) + break } data = packet.Data()[of:] @@ -541,7 +551,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { t.deleteMessage(m) if m.AssocMessage != nil { - m.AssocMessage.AssocMessage = nil + m.setAssocMessage(nil) } for _, pkt := range m.packets { @@ -575,8 +585,8 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { if !isIncoming { if responseRequest != nil { - message.AssocMessage = responseRequest - responseRequest.AssocMessage = message + message.setAssocMessage(responseRequest) + responseRequest.setAssocMessage(message) } else { t.respWithoutReq[packet.Ack] = packet.ID } @@ -587,17 +597,18 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { message.AddPacket(packet) // Handling Expect: 100-continue requests - if message.Is100Continue() { + if message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 { seq := packet.Seq + uint32(len(packet.Data)) t.seqWithData[seq] = packet.Ack message.DataSeq = seq + message.complete = false // In case if sequence packet came first for _, m := range t.messages { if m.Seq == seq { t.deleteMessage(m) if m.AssocMessage != nil { - message.AssocMessage = m.AssocMessage + message.setAssocMessage(m.AssocMessage) } // log.Println("2: Adding ack alias:", m.Ack, packet.Ack) t.ackAliases[m.Ack] = packet.Ack @@ -626,13 +637,13 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } // If message contains only single packet immediately dispatch it - if message.IsFinished() { + if message.complete { if isIncoming { // log.Println("I'm finished", string(message.Bytes()), message.ResponseID, t.messages) if t.trackResponse { if resp, ok := t.messages[message.ResponseID]; ok { t.dispatchMessage(message) - if resp.IsFinished() { + if resp.complete { t.dispatchMessage(resp) } } @@ -645,7 +656,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } if req, ok := t.messages[message.AssocMessage.ID()]; ok { - if req.IsFinished() { + if req.complete { t.dispatchMessage(req) t.dispatchMessage(message) } diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index 3f3b250..b66b23a 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -117,10 +117,10 @@ func TestShort100Continue(t *testing.T) { reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n")) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") @@ -140,7 +140,7 @@ func Test100ContinueWrongOrder(t *testing.T) { respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") @@ -159,15 +159,13 @@ func TestAlt100ContinueHeaderOrder(t *testing.T) { respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2) } - - func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) { var req, resp *TCPMessage for _, p := range packets { @@ -320,17 +318,17 @@ func TestRawListenerChunkedWrongOrder(t *testing.T) { reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n")) reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n")) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n")) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n")) + respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) // Should re-construct message from all possible combinations for i := 0; i < 6*5*4*3*2*1; i++ { - if i < 54 || i > 57 { - continue - } + // if i < 54 || i > 57 { + // continue + // } packets := permutation(i, []*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2}) @@ -349,7 +347,7 @@ func chunkedPostMessage() []*TCPPacket { reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n")) reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n")) - respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n")) + respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n")) return []*TCPPacket{ reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket, @@ -372,7 +370,7 @@ func postMessage() []*TCPPacket { return []*TCPPacket{ buildPacket(true, ack, seq, data), - buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n")), + buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n")), } } @@ -383,7 +381,7 @@ func getMessage() []*TCPPacket { return []*TCPPacket{ buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n")), - buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n")), + buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n")), } } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 5045f4d..3eca1b5 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -7,9 +7,9 @@ import ( "encoding/hex" "github.com/buger/gor/proto" "log" + "net" "strconv" "time" - "net" ) var _ = log.Println @@ -36,6 +36,15 @@ type TCPMessage struct { packets []*TCPPacket delChan chan *TCPMessage + + /* HTTP specific variables */ + methodType httpMethodType + bodyType httpBodyType + expectType httpExpectType + seqMissing bool + headerPacket int + contentLength int + complete bool } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted @@ -121,139 +130,294 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { t.DataAck = packet.OrigAck } } + + t.checkSeqIntegrity() + t.updateHeadersPacket() + t.updateMethodType() + t.updateBodyType() + t.checkIfComplete() + t.check100Continue() } // Check if there is missing packet -func (t *TCPMessage) isSeqMissing() bool { +func (t *TCPMessage) checkSeqIntegrity() { if len(t.packets) == 1 { - return false + t.seqMissing = false } for i, p := range t.packets { // If final packet if len(t.packets) == i+1 { - return false + t.seqMissing = false + return } np := t.packets[i+1] - if np.Seq != p.Seq+uint32(len(p.Data)) { - return true + nextSeq := p.Seq + uint32(len(p.Data)) + + if np.Seq != nextSeq { + if t.expectType == httpExpect100Continue { + if np.Seq != nextSeq+22 { + t.seqMissing = true + return + } + } else { + t.seqMissing = true + return + } } } - return false + t.seqMissing = false } -var EmptyLine = []byte("\r\n\r\n") -var ChunkEnd = []byte("0\r\n\r\n") +var bCLRF = []byte("\r\n") +var bEmptyLine = []byte("\r\n\r\n") +var bChunkEnd = []byte("0\r\n\r\n") -func (t *TCPMessage) isHeadersReceived() bool { - for _, p := range t.packets { - if bytes.LastIndex(p.Data, EmptyLine) != -1 { - return true +func (t *TCPMessage) updateHeadersPacket() { + if len(t.packets) == 1 { + t.headerPacket = -1 + } + + if t.headerPacket != -1 { + return + } + + if t.seqMissing { + return + } + + for i, p := range t.packets { + if bytes.LastIndex(p.Data, bEmptyLine) != -1 { + t.headerPacket = i + return } } - return false + return } // isMultipart returns true if message contains from multiple tcp packets -func (t *TCPMessage) IsFinished() bool { - payload := t.packets[0].Data - - if len(payload) < 4 { - return true +func (t *TCPMessage) checkIfComplete() { + if t.seqMissing || t.headerPacket == -1 { + return } - m := payload[:4] + if t.methodType == httpMethodNotFound { + return + } + + // Responses can be emitted only if we found request + if !t.IsIncoming && t.AssocMessage == nil { + return + } + + // If one GET, OPTIONS, or HEAD request + if t.methodType == httpMethodWithoutBody { + t.complete = true + } else { + switch t.bodyType { + case httpBodyEmpty: + t.complete = true + case httpBodyContentLength: + if t.contentLength == 0 || t.contentLength == t.BodySize() { + t.complete = true + } + case httpBodyChunked: + lastPacket := t.packets[len(t.packets)-1] + if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 { + t.complete = true + } + } + } +} + +type httpMethodType uint8 + +const ( + httpMethodNotSet httpMethodType = 0 + httpMethodWithBody httpMethodType = 1 + httpMethodWithoutBody httpMethodType = 2 + httpMethodNotFound httpMethodType = 3 +) + +var methodsWithBody = [][]byte{ + []byte("POST"), + []byte("PUT"), + []byte("PATCH"), + []byte("CONNECT"), +} + +func (t *TCPMessage) updateMethodType() { + // if there is cache + if t.methodType != httpMethodNotSet && t.methodType != httpMethodNotFound { + return + } + + d := t.packets[0].Data + + // Minimum length fo request: GET / HTTP/1.1\r\n + + if len(d) < 16 { + t.methodType = httpMethodNotFound + return + } if t.IsIncoming { - // If one GET, OPTIONS, or HEAD request - if bytes.Equal(m, []byte("GET ")) || bytes.Equal(m, []byte("OPTI")) || bytes.Equal(m, []byte("HEAD")) { - if !t.isSeqMissing() && t.isHeadersReceived() { - return true - } else { - return false + var method []byte + if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 { + method = d[:mIdx] + + // Check that after method we have absolute or relative path + switch d[mIdx+1] { + case '/', 'h', '*': + default: + t.methodType = httpMethodNotFound + return } } else { - // Sometimes header comes after the body :( - if bytes.Equal(m, []byte("POST")) || bytes.Equal(m, []byte("PUT ")) || bytes.Equal(m, []byte("PATC")) { + t.methodType = httpMethodNotFound + return + } - if t.isHeadersReceived() { - if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { - l, _ := strconv.Atoi(string(length)) - - // If content-length equal current body length - if l > 0 && l == t.BodySize() { - return true - } - } - } + for _, m := range methodsWithBody { + if len(m) == len(method) && bytes.Equal(m, method) { + t.methodType = httpMethodWithBody + return } } + + t.methodType = httpMethodWithoutBody } else { - // Request not found - // Can be because response came first or request request was just missing - if t.AssocMessage == nil { - return false + if !bytes.Equal(d[:6], []byte("HTTP/1")) { + t.methodType = httpMethodNotFound + return } - if !bytes.Equal(m, []byte("HTTP")) { - return false + t.methodType = httpMethodWithBody + } +} + +type httpBodyType uint8 + +const ( + httpBodyNotSet httpBodyType = 0 + httpBodyEmpty httpBodyType = 1 + httpBodyContentLength httpBodyType = 2 + httpBodyChunked httpBodyType = 3 +) + +func (t *TCPMessage) updateBodyType() { + // if there is cache + if t.bodyType != httpBodyNotSet { + return + } + + // Headers not received + if t.headerPacket == -1 { + return + } + + switch t.methodType { + case httpMethodNotFound: + return + case httpMethodWithoutBody: + t.bodyType = httpBodyEmpty + return + case httpMethodWithBody: + var lengthB, encB []byte + + for _, p := range t.packets[:t.headerPacket+1] { + lengthB = proto.Header(p.Data, []byte("Content-Length")) + + if len(lengthB) > 0 { + break + } } - if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { - if length[0] == '0' { - return true - } - - l, _ := strconv.Atoi(string(length)) - - // If content-length equal current body length - if l > 0 && l == t.BodySize() { - return true - } + if len(lengthB) > 0 { + t.bodyType = httpBodyContentLength + t.contentLength, _ = strconv.Atoi(string(lengthB)) + return } else { - if enc := proto.Header(payload, []byte("Transfer-Encoding")); len(enc) == 0 { - return true - } else { - if len(t.packets) > 1 && bytes.LastIndex(t.packets[len(t.packets)-1].Data, ChunkEnd) != -1 { - return true + for _, p := range t.packets[:t.headerPacket+1] { + encB = proto.Header(p.Data, []byte("Transfer-Encoding")) + + if len(encB) > 0 { + t.bodyType = httpBodyChunked + return } } } } - return false + t.bodyType = httpBodyEmpty } +type httpExpectType uint8 + +const ( + httpExpectNotSet httpExpectType = 0 + httpExpectEmpty httpExpectType = 1 + httpExpect100Continue httpExpectType = 2 +) + var bExpectHeader = []byte("Expect:") var bExpect100Value = []byte("100-continue") -var bPOST = []byte("POST") -var bCRLFx2 = []byte("\r\n\r\n") -func (t *TCPMessage) Is100Continue() bool { - d := t.packets[0].Data - - if len(d) < 25 { - return false +func (t *TCPMessage) check100Continue() { + if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 { + return } - if !bytes.Equal(d[0:4], bPOST) { - return false + if t.methodType != httpMethodWithBody { + return } + if t.seqMissing || t.headerPacket == -1 { + return + } + + last := t.packets[len(t.packets)-1] // reading last 4 bytes for double CRLF - if !bytes.Equal(d[len(d)-4:], bCRLFx2) { - return false + if !bytes.HasSuffix(last.Data, bEmptyLine) { + return } - // look for an expect:100-continue header - if !bytes.Equal(bExpect100Value, proto.Header(d, bExpectHeader)) { - return false + for _, p := range t.packets[:t.headerPacket+1] { + if h := proto.Header(p.Data, bExpectHeader); len(h) > 0 { + if bytes.Equal(bExpect100Value, h) { + t.expectType = httpExpect100Continue + } + return + } } - return true + t.expectType = httpExpectEmpty +} + +func (t *TCPMessage) setAssocMessage(m *TCPMessage) { + t.AssocMessage = m + t.checkIfComplete() +} + +// UpdateResponseAck should be called after packet is added +func (t *TCPMessage) UpdateResponseAck() uint32 { + lastPacket := t.packets[len(t.packets)-1] + respAck := lastPacket.Seq + uint32(len(lastPacket.Data)) + + if t.ResponseAck != respAck { + t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data)) + + // We swappwed src and dst port + copy(t.ResponseID[:16], lastPacket.Addr) + copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port + copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port + binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck) + } + + return t.ResponseAck } func (t *TCPMessage) UUID() []byte { @@ -276,28 +440,10 @@ func (t *TCPMessage) UUID() []byte { return uuid } -// UpdateResponseAck should be called after packet is added -func (t *TCPMessage) UpdateResponseAck() uint32 { - lastPacket := t.packets[len(t.packets)-1] - respAck := lastPacket.Seq + uint32(len(lastPacket.Data)) - - if t.ResponseAck != respAck { - t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data)) - - // We swappwed src and dst port - copy(t.ResponseID[:16], lastPacket.Addr) - copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port - copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port - binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck) - } - - return t.ResponseAck -} - func (t *TCPMessage) ID() tcpID { return t.packets[0].ID } func (t *TCPMessage) IP() net.IP { return net.IP(t.packets[0].Addr) -} \ No newline at end of file +} diff --git a/raw_socket_listener/tcp_message_test.go b/raw_socket_listener/tcp_message_test.go index 2b90918..7220385 100644 --- a/raw_socket_listener/tcp_message_test.go +++ b/raw_socket_listener/tcp_message_test.go @@ -80,77 +80,45 @@ func TestTCPMessageSize(t *testing.T) { } } -func TestTCPMessageIsFinished(t *testing.T) { - methodsWithoutBodies := []string{"GET", "OPTIONS", "HEAD"} +func TestTCPMessageIsComplete(t *testing.T) { + testCases := []struct { + direction bool + payload string + assocMessage bool + expectedCompleted bool + }{ + {true, "GET / HTTP/1.1\r\n\r\n", false, true}, + {true, "HEAD / HTTP/1.1\r\n\r\n", false, true}, + {false, "HTTP/1.1 200 OK\r\n\r\n", true, true}, + {true, "POST / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true}, + {true, "PUT / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true}, + {false, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", true, true}, + {false, "HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na", true, true}, + {false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", true, true}, - for _, m := range methodsWithoutBodies { - msg := buildMessage(buildPacket(true, 1, 1, []byte(m+" / HTTP/1.1\r\n\r\n"))) + // chunked not finished + {false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", true, false}, - if !msg.IsFinished() { - t.Error(m, " request should be finished") + // content-length != actual length + {true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na", false, false}, + {false, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na", true, false}, + // non-valid http request + {true, "UNKNOWN asd HTTP/1.1\r\n\r\n", false, false}, + + // response without associated request + {false, "HTTP/1.1 200 OK\r\n\r\n", false, false}, + } + + for _, tc := range testCases { + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + if tc.assocMessage { + msg.AssocMessage = &TCPMessage{} } - } + msg.checkIfComplete() - methodsWithBodies := []string{"POST", "PUT", "PATCH"} - - for _, m := range methodsWithBodies { - msg := buildMessage(buildPacket(true, 1, 1, []byte(m+" / HTTP/1.1\r\nContent-Length: 1\r\n\r\na"))) - - if !msg.IsFinished() { - t.Error(m, " should be finished as body length == content length") + if msg.complete != tc.expectedCompleted { + t.Errorf("Payload %s: Expected %t, got %t.", tc.payload, tc.expectedCompleted, msg.complete) } - - msg = buildMessage(buildPacket(true, 1, 1, []byte(m+" / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"))) - - if msg.IsFinished() { - t.Error(m, " should not be finished as body length != content length") - } - } - - msg := buildMessage(buildPacket(true, 1, 1, []byte("UNKNOWN / HTTP/1.1\r\n\r\n"))) - if msg.IsFinished() { - t.Error("non http or wrong methods considered as not finished") - } - - // Responses - msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"))) - msg.AssocMessage = &TCPMessage{} - if !msg.IsFinished() { - t.Error("Should mark simple response as finished") - } - - msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"))) - msg.AssocMessage = nil - if msg.IsFinished() { - t.Error("Should not mark responses without associated requests") - } - - msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"))) - msg.AssocMessage = &TCPMessage{} - - if msg.IsFinished() { - t.Error("Should mark chunked response as non finished") - } - - msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"))) - msg.AssocMessage = &TCPMessage{} - - if !msg.IsFinished() { - t.Error("Should mark Content-Length: 0 respones as finished") - } - - msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na"))) - msg.AssocMessage = &TCPMessage{} - - if !msg.IsFinished() { - t.Error("Should mark valid Content-Length respones as finished") - } - - msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na"))) - msg.AssocMessage = &TCPMessage{} - - if msg.IsFinished() { - t.Error("Should not mark not valid Content-Length respones as finished") } } @@ -160,32 +128,90 @@ func TestTCPMessageIsSeqMissing(t *testing.T) { p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a")) msg := buildMessage(p1) - if msg.isSeqMissing() { + if msg.seqMissing { t.Error("Should be complete if have only 1 packet") } msg.AddPacket(p3) - if !msg.isSeqMissing() { + if !msg.seqMissing { t.Error("Should be incomplete because missing middle component") } msg.AddPacket(p2) - if msg.isSeqMissing() { + if msg.seqMissing { t.Error("Should be complete once missing packet added") } } func TestTCPMessageIsHeadersReceived(t *testing.T) { - p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n")) + p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n")) p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n")) msg := buildMessage(p1) - if msg.isHeadersReceived() { - t.Error("Should be complete if have only 1 packet") + if msg.headerPacket == -1 { + t.Error("Should be complete if have only 1 packet", msg.headerPacket) } msg.AddPacket(p2) - if !msg.isHeadersReceived() { + if msg.headerPacket == -1 { t.Error("Should found double new line: headers received") } + + msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"))) + if msg.headerPacket != -1 { + t.Error("Should not find headers end") + } +} + +func TestTCPMessageMethodType(t *testing.T) { + testCases := []struct { + direction bool + payload string + expectedMethodType httpMethodType + }{ + {true, "GET / HTTP/1.1\r\n\r\n", httpMethodWithoutBody}, + {true, "GET * HTTP/1.1\r\n\r\n", httpMethodWithoutBody}, + {true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodWithoutBody}, + {true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodWithoutBody}, + {true, "POST / HTTP/1.1\r\n\r\n", httpMethodWithBody}, + {true, "PUT / HTTP/1.1\r\n\r\n", httpMethodWithBody}, + {true, "GET zxc HTTP/1.1\r\n\r\n", httpMethodNotFound}, + {true, "GET / HTTP\r\n\r\n", httpMethodNotFound}, + {true, "VERYLONGMETHOD / HTTP/1.1\r\n\r\n", httpMethodNotFound}, + {false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodWithBody}, + {false, "HTTP /1.1 200 OK\r\n\r\n", httpMethodNotFound}, + } + + for _, tc := range testCases { + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + + if msg.methodType != tc.expectedMethodType { + t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType) + } + } +} + +func TestTCPMessageBodyType(t *testing.T) { + testCases := []struct { + direction bool + payload string + expectedBodyType httpBodyType + }{ + {true, "GET / HTTP/1.1\r\n\r\n", httpBodyEmpty}, + {true, "POST / HTTP/1.1\r\n\r\n", httpBodyEmpty}, + {true, "POST / HTTP/1.1\r\nUser-Agent: zxc\r\n\r\n", httpBodyEmpty}, + {false, "HTTP/1.1 200 OK\r\n\r\n", httpBodyEmpty}, + {true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength}, + {false, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength}, + {true, "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked}, + {false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked}, + } + + for _, tc := range testCases { + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + + if msg.bodyType != tc.expectedBodyType { + t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType) + } + } } From 37650df2c86e7b2ae752c46b3ff03bf81f58436e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 29 Jun 2016 17:29:09 +0300 Subject: [PATCH 17/79] Refactor #308 --- input_dummy.go | 4 +- input_file.go | 243 ++++++++++++++------------- input_file_test.go | 8 +- input_http.go | 2 +- input_raw.go | 4 +- output_http.go | 5 +- proto/proto.go | 14 +- protocol.go | 21 ++- raw_socket_listener/listener.go | 42 ++--- raw_socket_listener/listener_test.go | 2 - raw_socket_listener/tcp_message.go | 4 +- test_input.go | 2 +- 12 files changed, 187 insertions(+), 164 deletions(-) diff --git a/input_dummy.go b/input_dummy.go index a2b4674..beb1f9c 100644 --- a/input_dummy.go +++ b/input_dummy.go @@ -34,10 +34,10 @@ func (i *DummyInput) emit() { select { case <-ticker.C: uuid := uuid() - reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano()) + reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano(), -1) i.data <- append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...) - resh := payloadHeader(ResponsePayload, uuid, 1) + resh := payloadHeader(ResponsePayload, uuid, time.Now().UnixNano()+1, 1) i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...) } } diff --git a/input_file.go b/input_file.go index 8170179..37e2b6a 100644 --- a/input_file.go +++ b/input_file.go @@ -9,35 +9,109 @@ import ( "log" "os" "path/filepath" - "strings" "strconv" + "strings" + "sync" "time" ) -// FileInput can read requests generated by FileOutput -type FileInput struct { - data chan []byte - exit chan bool - path string - fileInputReaders []*fileInputReader - speedFactor float64 - loop bool -} - - type fileInputReader struct { reader *bufio.Reader - meta [][]byte data []byte file *os.File timestamp int64 } +func (f *fileInputReader) parseNext() error { + payloadSeparatorAsBytes := []byte(payloadSeparator) + var buffer bytes.Buffer + + for { + line, err := f.reader.ReadBytes('\n') + + if err != nil { + if err != io.EOF { + log.Println(err) + return err + } + + if err == io.EOF { + f.file.Close() + f.file = nil + return err + } + } + + if bytes.Equal(payloadSeparatorAsBytes[1:], line) { + asBytes := buffer.Bytes() + meta := payloadMeta(asBytes) + + f.timestamp, _ = strconv.ParseInt(string(meta[2]), 10, 64) + f.data = asBytes[:len(asBytes)-1] + + return nil + } + + buffer.Write(line) + } + + return nil +} + +func (f *fileInputReader) ReadPayload() []byte { + defer f.parseNext() + + return f.data +} +func (f *fileInputReader) Close() error { + if f.file != nil { + f.file.Close() + } + + return nil +} + +func NewFileInputReader(path string) *fileInputReader { + file, err := os.Open(path) + + if err != nil { + log.Println(err) + return nil + } + + r := &fileInputReader{file: file} + if strings.HasSuffix(path, ".gz") { + gzReader, err := gzip.NewReader(file) + if err != nil { + log.Println(err) + return nil + } + r.reader = bufio.NewReader(gzReader) + } else { + r.reader = bufio.NewReader(file) + } + + r.parseNext() + + return r +} + +// FileInput can read requests generated by FileOutput +type FileInput struct { + mu sync.Mutex + data chan []byte + exit chan bool + path string + readers []*fileInputReader + speedFactor float64 + loop bool +} + // NewFileInput constructor for FileInput. Accepts file path as argument. func NewFileInput(path string, loop bool) (i *FileInput) { i = new(FileInput) - i.data = make(chan []byte) - i.exit = make(chan bool) + i.data = make(chan []byte, 1000) + i.exit = make(chan bool, 1) i.path = path i.speedFactor = 1 i.loop = loop @@ -58,6 +132,9 @@ func (_ *NextFileNotFound) Error() string { } func (i *FileInput) init() (err error) { + defer i.mu.Unlock() + i.mu.Lock() + var matches []string if matches, err = filepath.Glob(i.path); err != nil { @@ -70,24 +147,10 @@ func (i *FileInput) init() (err error) { return errors.New("No matching files") } - i.fileInputReaders = make([]*fileInputReader, len(matches)) + i.readers = make([]*fileInputReader, len(matches)) for idx, p := range matches { - file, _ := os.Open(p) - fileInputReader := &fileInputReader{} - fileInputReader.file = file - if strings.HasSuffix(p, ".gz") { - gzReader, err := gzip.NewReader(file) - if err != nil { - log.Fatal(err) - } - fileInputReader.reader = bufio.NewReader(gzReader) - } else { - fileInputReader.reader = bufio.NewReader(file) - } - - fileInputReader.readNextInput() - i.fileInputReaders[idx] = fileInputReader + i.readers[idx] = NewFileInputReader(p) } return nil @@ -104,123 +167,71 @@ func (i *FileInput) String() string { return "File input: " + i.path } -func (f *fileInputReader) readNextInput() { - nextInput := f.nextInput() - f.parseNextInput(nextInput) -} - -func (f *fileInputReader) parseNextInput(input []byte) { - if (input != nil) { - f.meta = payloadMeta(input) - f.timestamp, _ = strconv.ParseInt(string(f.meta[2]), 10, 64) - f.data = input - } -} - -func (f *fileInputReader) nextInput() []byte { - payloadSeparatorAsBytes := []byte(payloadSeparator) - var buffer bytes.Buffer - - for { - line, err := f.reader.ReadBytes('\n') - - if err != nil { - if err != io.EOF { - log.Fatal(err) - } - - if err == io.EOF { - f.file.Close() - f.file = nil - return nil - } - } - - if bytes.Equal(payloadSeparatorAsBytes[1:], line) { - asBytes := buffer.Bytes() - - // Bytes() returns only pointer, so to remove data-race copy the data to an array - newBuf := make([]byte, len(asBytes) - 1) - copy(newBuf, asBytes) - return newBuf - } - - buffer.Write(line) - } -} - -func (i *FileInput) nextInputReader() *fileInputReader { - var nextFileInputReader *fileInputReader - for _, fileInputReader := range i.fileInputReaders { - if fileInputReader.file == nil { +// Find reader with smallest timestamp e.g next payload in row +func (i *FileInput) nextReader() (next *fileInputReader) { + for _, r := range i.readers { + if r == nil || r.file == nil { continue } - if fileInputReader.meta[0][0] == ResponsePayload { - return fileInputReader - } - - if nextFileInputReader == nil || nextFileInputReader.timestamp > fileInputReader.timestamp { - nextFileInputReader = fileInputReader + if next == nil || r.timestamp < next.timestamp { + next = r continue } } - return nextFileInputReader; + return } func (i *FileInput) emit() { var lastTime int64 = -1 for { - fileInputReader := i.nextInputReader() + select { + case <-i.exit: + return + default: + } - if fileInputReader == nil { + reader := i.nextReader() + + if reader == nil { if i.loop { i.init() lastTime = -1 continue } else { - break; + break } } - if fileInputReader.meta[0][0] == RequestPayload { - lastTime = i.simulateRequestDelay(fileInputReader, lastTime) + if lastTime != -1 { + diff := reader.timestamp - lastTime + lastTime = reader.timestamp + + if i.speedFactor != 1 { + diff = int64(float64(diff) / i.speedFactor) + } + + time.Sleep(time.Duration(diff)) + } else { + lastTime = reader.timestamp } - select { - case <-i.exit: - for _, fileInputReader := range i.fileInputReaders { - if fileInputReader.file != nil { - fileInputReader.file.Close() - } - } - break - case i.data <- fileInputReader.data: - fileInputReader.readNextInput() - } + i.data <- reader.ReadPayload() } log.Printf("FileInput: end of file '%s'\n", i.path) } -func (i*FileInput) simulateRequestDelay(fileInputReader *fileInputReader, lastTime int64) int64 { - if lastTime != -1 { - timeDiff := fileInputReader.timestamp - lastTime +func (i *FileInput) Close() error { + defer i.mu.Unlock() + i.mu.Lock() - if i.speedFactor != 1 { - timeDiff = int64(float64(timeDiff) / i.speedFactor) - } + i.exit <- true - time.Sleep(time.Duration(timeDiff)) + r.Close() } - return fileInputReader.timestamp -} - -func (i *FileInput) Close() error { - i.exit <- true return nil } - diff --git a/input_file_test.go b/input_file_test.go index 61c18cd..9029611 100644 --- a/input_file_test.go +++ b/input_file_test.go @@ -131,7 +131,7 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) { func TestInputFileRequestsWithLatency(t *testing.T) { rnd := rand.Int63() - file, _ := os.OpenFile(fmt.Sprintf("/tmp/%d", rnd), os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0660) + file, _ := os.OpenFile(fmt.Sprintf("/tmp/%d", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) defer file.Close() file.Write([]byte("1 1 100000000\nrequest1")) @@ -156,7 +156,7 @@ func TestInputFileRequestsWithLatency(t *testing.T) { t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency) } - if realLatency > expectedLatency + 10000000 { + if realLatency > expectedLatency+10000000 { t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency) } @@ -180,7 +180,7 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) { file2.Write([]byte("1 3 2\nrequest3")) file2.Write([]byte(payloadSeparator)) file2.Write([]byte("2 3 2\nresponse3")) - file2.Write([]byte(payloadSeparator)) + file2.Write([]byte(payloadSeparator)) file2.Write([]byte("1 4 4\nrequest4")) file2.Write([]byte(payloadSeparator)) file2.Write([]byte("2 4 4\nresponse4")) @@ -295,11 +295,9 @@ func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool { } for i, payload := range other { - if !bytes.Equal(expectedCaptureFile.data[i], payload) { return false } - } return true diff --git a/input_http.go b/input_http.go index 33263b9..f3a6419 100644 --- a/input_http.go +++ b/input_http.go @@ -29,7 +29,7 @@ func NewHTTPInput(address string) (i *HTTPInput) { func (i *HTTPInput) Read(data []byte) (int, error) { buf := <-i.data - header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1) copy(data[0:len(header)], header) copy(data[len(header):], buf) diff --git a/input_raw.go b/input_raw.go index 7b92096..26b3dd0 100644 --- a/input_raw.go +++ b/input_raw.go @@ -50,12 +50,12 @@ func (i *RAWInput) Read(data []byte) (int, error) { var header []byte if msg.IsIncoming { - header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano()) + header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano(), -1) if len(i.realIPHeader) > 0 { buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String())) } } else { - header = payloadHeader(ResponsePayload, msg.UUID(), msg.End.UnixNano()-msg.AssocMessage.Start.UnixNano()) + header = payloadHeader(ResponsePayload, msg.UUID(), msg.AssocMessage.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.Start.UnixNano()) } copy(data[0:len(header)], header) diff --git a/output_http.go b/output_http.go index 97423a8..d442b83 100644 --- a/output_http.go +++ b/output_http.go @@ -14,6 +14,7 @@ type response struct { payload []byte uuid []byte roundTripTime int64 + startedAt int64 } // HTTPOutputConfig struct for holding http output configuration @@ -178,7 +179,7 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { Debug("[OUTPUT-HTTP] Received response:", string(resp.payload)) - header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime) + header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime) copy(data[0:len(header)], header) copy(data[len(header):], resp.payload) @@ -206,7 +207,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { } if o.config.TrackResponses { - o.responses <- response{resp, uuid, stop.UnixNano() - start.UnixNano()} + o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()} } if o.elasticSearch != nil { diff --git a/proto/proto.go b/proto/proto.go index 40e59b1..186926f 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -133,7 +133,7 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart = headerStart + len(name) + 1 // Skip ":" after header name headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n') - for valueStart < headerEnd { // Ignore empty space after ':' + for valueStart < headerEnd { // Ignore empty space after ':' if payload[valueStart] == ' ' { valueStart++ } else { @@ -148,7 +148,7 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, } // ignore empty space at end of header value - for valueStart < valueEnd { + for valueStart < valueEnd { if payload[valueEnd-1] == ' ' { valueEnd-- } else { @@ -170,7 +170,7 @@ func Header(payload, name []byte) []byte { // SetHeader sets header value. If header not found it creates new one. // Returns modified request payload func SetHeader(payload, name, value []byte) []byte { - _, hs, _, vs, ve := header(payload, name) + _, hs, _, vs, ve := header(payload, name) if hs != -1 { // If header found we just replace its value @@ -196,12 +196,12 @@ func AddHeader(payload, name, value []byte) []byte { // DelHeader takes http payload and removes header name from headers section // Returns modified request payload -func DeleteHeader(payload, name[]byte) []byte { +func DeleteHeader(payload, name []byte) []byte { _, hs, he, _, _ := header(payload, name) if hs != -1 { - newHeader := make([]byte, len(payload) - (he - hs) - 1) + newHeader := make([]byte, len(payload)-(he-hs)-1) copy(newHeader[:hs], payload[:hs]) - copy(newHeader[hs:], payload[he + 1:]) + copy(newHeader[hs:], payload[he+1:]) return newHeader } return payload @@ -322,7 +322,7 @@ func Status(payload []byte) []byte { } var httpMethods []string = []string{ - "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", /* custom methods */"BAN", "PURG", + "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN" /* custom methods */, "BAN", "PURG", } func IsHTTPPayload(payload []byte) bool { diff --git a/protocol.go b/protocol.go index 1b4c484..50d6943 100644 --- a/protocol.go +++ b/protocol.go @@ -42,13 +42,24 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err } // Timing is request start or round-trip time, depending on payloadType -func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte) { - sTime := strconv.FormatInt(timing, 10) +func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) { + var sTime, sLatency string + + sTime = strconv.FormatInt(timing, 10) + if latency != -1 { + sLatency = strconv.FormatInt(latency, 10) + } //Example: // 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n // `+ 1` indicates space characters or end of line - header = make([]byte, 1+1+len(uuid)+1+len(sTime)+1) + headerLen := 1 + 1 + len(uuid) + 1 + len(sTime) + 1 + + if latency != -1 { + headerLen += len(sLatency) + 1 + } + + header = make([]byte, headerLen) header[0] = payloadType header[1] = ' ' header[2+len(uuid)] = ' ' @@ -57,6 +68,10 @@ func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte) copy(header[2:], uuid) copy(header[3+len(uuid):], sTime) + if latency != -1 { + copy(header[4+len(uuid)+len(sTime):], sLatency) + } + return header } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 3cd5e70..5a492d8 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -16,6 +16,7 @@ import ( "bytes" "encoding/binary" "fmt" + "github.com/buger/gor/proto" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/google/gopacket/pcap" @@ -28,7 +29,6 @@ import ( "strings" "sync" "time" - "github.com/buger/gor/proto" ) var _ = fmt.Println @@ -303,7 +303,7 @@ func (t *Listener) readPcap() { for i, addr := range device.Addresses { bpfDstHost += "dst host " + addr.IP.String() bpfSrcHost += "src host " + addr.IP.String() - if i != len(device.Addresses) - 1 { + if i != len(device.Addresses)-1 { bpfDstHost += " or " bpfSrcHost += " or " } @@ -330,9 +330,9 @@ func (t *Listener) readPcap() { // Special case for tunnel interface https://github.com/google/gopacket/issues/99 if handle.LinkType() == 12 { - decoder = layers.LayerTypeIPv4 + decoder = layers.LayerTypeIPv4 } else { - decoder = handle.LinkType() + decoder = handle.LinkType() } source := gopacket.NewPacketSource(handle, decoder) @@ -355,23 +355,23 @@ func (t *Listener) readPcap() { // We should remove network layer before parsing TCP/IP data var of int switch decoder { - case layers.LinkTypeEthernet: - of = 14 - case layers.LinkTypePPP: - of = 1 - case layers.LinkTypeFDDI: - of = 13 - case layers.LinkTypeNull: - of = 4 - case layers.LinkTypeLoop: - of = 4 - case layers.LinkTypeRaw: - of = 0 - case layers.LinkTypeLinuxSLL: - of = 16 - default: - log.Println("Unknown packet layer", packet) - break + case layers.LinkTypeEthernet: + of = 14 + case layers.LinkTypePPP: + of = 1 + case layers.LinkTypeFDDI: + of = 13 + case layers.LinkTypeNull: + of = 4 + case layers.LinkTypeLoop: + of = 4 + case layers.LinkTypeRaw: + of = 0 + case layers.LinkTypeLinuxSLL: + of = 16 + default: + log.Println("Unknown packet layer", packet) + break } data = packet.Data()[of:] diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index 3f3b250..19afb74 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -166,8 +166,6 @@ func TestAlt100ContinueHeaderOrder(t *testing.T) { testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2) } - - func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) { var req, resp *TCPMessage for _, p := range packets { diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 5045f4d..4300233 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -7,9 +7,9 @@ import ( "encoding/hex" "github.com/buger/gor/proto" "log" + "net" "strconv" "time" - "net" ) var _ = log.Println @@ -300,4 +300,4 @@ func (t *TCPMessage) ID() tcpID { func (t *TCPMessage) IP() net.IP { return net.IP(t.packets[0].Addr) -} \ No newline at end of file +} diff --git a/test_input.go b/test_input.go index cd146c5..0f990e7 100644 --- a/test_input.go +++ b/test_input.go @@ -22,7 +22,7 @@ func NewTestInput() (i *TestInput) { func (i *TestInput) Read(data []byte) (int, error) { buf := <-i.data - header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1) copy(data[0:len(header)], header) copy(data[len(header):], buf) From 7c3333643362e702eb61a0be47dc0b07805d0bb2 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 29 Jun 2016 17:29:36 +0300 Subject: [PATCH 18/79] Add missing change --- input_file.go | 1 + 1 file changed, 1 insertion(+) diff --git a/input_file.go b/input_file.go index 37e2b6a..6dfffc4 100644 --- a/input_file.go +++ b/input_file.go @@ -230,6 +230,7 @@ func (i *FileInput) Close() error { i.exit <- true + for _, r := range i.readers { r.Close() } From 5a0c03ec339a4d5661d91ff353bb3daf3059e959 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 30 Jun 2016 14:13:27 +0300 Subject: [PATCH 19/79] Fix BodySize for multi-packet headers --- .gitignore | 2 + Makefile | 3 ++ input_raw.go | 7 ++++ plugins.go | 2 + raw_socket_listener/listener.go | 52 +++++++++++++++++++++++++ raw_socket_listener/tcp_message.go | 7 ++-- raw_socket_listener/tcp_message_test.go | 29 ++++++++++++++ 7 files changed, 99 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index de9ee5a..4c1635a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ gor *.mprof + +*.pcap diff --git a/Makefile b/Makefile index 7e1df0d..f4b6249 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,9 @@ run-arg: file-server: go run $(SOURCE) file-server $(FADDR) +readpcap: + go run $(SOURCE) --input-raw $(FILE) --input-raw-engine pcap_file --output-stdout + record: $(RUN) go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug diff --git a/input_raw.go b/input_raw.go index 26b3dd0..0236a2c 100644 --- a/input_raw.go +++ b/input_raw.go @@ -24,6 +24,7 @@ type RAWInput struct { const ( EngineRawSocket = 1 << iota EnginePcap + EnginePcapFile ) // NewRAWInput constructor for RAWInput. Accepts address with port as argument. @@ -69,6 +70,12 @@ func (i *RAWInput) listen(address string) { host, port, err := net.SplitHostPort(address) + if i.engine == EnginePcapFile { + host = address + port = "1" + err = nil + } + if err != nil { log.Fatal("input-raw: error while parsing address", err) } diff --git a/plugins.go b/plugins.go index dac9def..c645d48 100644 --- a/plugins.go +++ b/plugins.go @@ -98,6 +98,8 @@ func InitPlugins() { engine := EnginePcap if Settings.inputRAWEngine == "raw_socket" { engine = EngineRawSocket + } else if Settings.inputRAWEngine == "pcap_file" { + engine = EnginePcapFile } for _, options := range Settings.inputRAW { diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 88066d2..0cfffa6 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -81,6 +81,7 @@ type request struct { const ( EngineRawSocket = 1 << iota EnginePcap + EnginePcapFile ) // NewListener creates and initializes new Listener object @@ -118,6 +119,8 @@ func NewListener(addr string, port string, engine int, trackResponse bool, expir go l.readRAWSocket() case EnginePcap: go l.readPcap() + case EnginePcapFile: + go l.readPcapFile() default: log.Fatal("Unknown traffic interception engine:", engine) } @@ -171,6 +174,8 @@ func (t *Listener) dispatchMessage(message *TCPMessage) { return } + log.Println("MESSAGE:", message, message.BodySize(), message.contentLength, message.methodType, message.bodyType) + t.deleteMessage(message) if message.methodType == httpMethodNotFound { @@ -466,6 +471,53 @@ func (t *Listener) readPcap() { t.readyCh <- true } +func (t *Listener) readPcapFile() { + if handle, err := pcap.OpenOffline(t.addr); err != nil { + log.Fatal(err) + } else { + t.readyCh <- true + packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) + + for { + packet, err := packetSource.NextPacket() + if err == io.EOF { + break + } else if err != nil { + log.Println("Error:", err) + continue + } + + var addr, data []byte + + if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil { + tcp, _ := tcpLayer.(*layers.TCP) + data = append(tcp.LayerContents(), tcp.LayerPayload()...) + copy(data[2:4], []byte{0, 1}) + } else { + log.Println("Can't find TCP layer", packet) + continue + } + + if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil { + ip, _ := ipLayer.(*layers.IPv4) + addr = ip.SrcIP + } else if ipLayer = packet.Layer(layers.LayerTypeIPv6); ipLayer != nil { + ip, _ := ipLayer.(*layers.IPv6) + addr = ip.SrcIP + } else { + log.Println("Can't find IP layer", packet) + continue + } + + newBuf := make([]byte, len(data)+16) + copy(newBuf[:16], addr) + copy(newBuf[16:], data) + + t.packetsChan <- newBuf + } + } +} + func (t *Listener) readRAWSocket() { conn, e := net.ListenPacket("ip:tcp", t.addr) t.conn = conn diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 3eca1b5..0f78fd8 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -66,13 +66,13 @@ func (t *TCPMessage) Bytes() (output []byte) { // Size returns total body size func (t *TCPMessage) BodySize() (size int) { - if len(t.packets) == 0 { + if len(t.packets) == 0 || t.headerPacket == -1 { return 0 } - size += len(proto.Body(t.packets[0].Data)) + size += len(proto.Body(t.packets[t.headerPacket].Data)) - for _, p := range t.packets[1:] { + for _, p := range t.packets[t.headerPacket + 1:] { size += len(p.Data) } @@ -266,6 +266,7 @@ func (t *TCPMessage) updateMethodType() { if t.IsIncoming { var method []byte + if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 { method = d[:mIdx] diff --git a/raw_socket_listener/tcp_message_test.go b/raw_socket_listener/tcp_message_test.go index 7220385..0af3bd9 100644 --- a/raw_socket_listener/tcp_message_test.go +++ b/raw_socket_listener/tcp_message_test.go @@ -215,3 +215,32 @@ func TestTCPMessageBodyType(t *testing.T) { } } } + + +func TestTCPMessageBodySize(t *testing.T) { + testCases := []struct { + direction bool + payloads []string + expectedSize int + }{ + {true, []string{"GET / HTTP/1.1\r\n\r\n"}, 0}, + {true, []string{"POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab"}, 2}, + {true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\nab"}, 2}, + {true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\n", "ab"}, 2}, + } + + for _, tc := range testCases { + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]))) + + if len(tc.payloads) > 1 { + for _, p := range tc.payloads[1:] { + seq := uint32(1 + msg.Size()) + msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p))) + } + } + + if msg.BodySize() != tc.expectedSize { + t.Errorf("Expected %d, got %d", tc.expectedSize, msg.BodySize()) + } + } +} From f17f51390a1478f1dacbe4381a32ea5c2fcea12e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 1 Jul 2016 20:01:25 +0300 Subject: [PATCH 20/79] Fix 100-continue with multi-packet headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also adds —output-null for testing purpose --- Makefile | 4 ++-- output_null.go | 18 ++++++++++++++++++ plugins.go | 4 ++++ raw_socket_listener/listener.go | 19 ++++++++++--------- settings.go | 3 +++ 5 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 output_null.go diff --git a/Makefile b/Makefile index f4b6249..c0c1e45 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_null.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go SOURCE_PATH = /go/src/github.com/buger/gor/ PORT = 8000 FADDR = :8000 @@ -77,7 +77,7 @@ file-server: go run $(SOURCE) file-server $(FADDR) readpcap: - go run $(SOURCE) --input-raw $(FILE) --input-raw-engine pcap_file --output-stdout + go run $(SOURCE) --input-raw $(FILE) --input-raw-engine pcap_file --output-null record: $(RUN) go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug diff --git a/output_null.go b/output_null.go new file mode 100644 index 0000000..14867a2 --- /dev/null +++ b/output_null.go @@ -0,0 +1,18 @@ +package main + +// NullOutput used for debugging, prints nothing +type NullOutput struct { +} + +// NullOutput constructor for NullOutput +func NewNullOutput() (o *NullOutput) { + return new(NullOutput) +} + +func (o *NullOutput) Write(data []byte) (int, error) { + return len(data), nil +} + +func (o *NullOutput) String() string { + return "Null Output" +} diff --git a/plugins.go b/plugins.go index c645d48..1a0f6db 100644 --- a/plugins.go +++ b/plugins.go @@ -95,6 +95,10 @@ func InitPlugins() { registerPlugin(NewDummyOutput) } + if Settings.outputNull { + registerPlugin(NewNullOutput) + } + engine := EnginePcap if Settings.inputRAWEngine == "raw_socket" { engine = EngineRawSocket diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 0cfffa6..f639254 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -174,14 +174,8 @@ func (t *Listener) dispatchMessage(message *TCPMessage) { return } - log.Println("MESSAGE:", message, message.BodySize(), message.contentLength, message.methodType, message.bodyType) - t.deleteMessage(message) - if message.methodType == httpMethodNotFound { - return - } - if !message.complete { if !message.IsIncoming { delete(t.respAliases, message.Ack) @@ -494,7 +488,6 @@ func (t *Listener) readPcapFile() { data = append(tcp.LayerContents(), tcp.LayerPayload()...) copy(data[2:4], []byte{0, 1}) } else { - log.Println("Can't find TCP layer", packet) continue } @@ -505,7 +498,15 @@ func (t *Listener) readPcapFile() { ip, _ := ipLayer.(*layers.IPv6) addr = ip.SrcIP } else { - log.Println("Can't find IP layer", packet) + // log.Println("Can't find IP layer", packet) + continue + } + + dataOffset := (data[12] & 0xF0) >> 4 + + // We need only packets with data inside + // Check that the buffer is larger than the size of the TCP header + if len(data) <= int(dataOffset*4) { continue } @@ -650,7 +651,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { // Handling Expect: 100-continue requests if message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 { - seq := packet.Seq + uint32(len(packet.Data)) + seq := packet.Seq + uint32(message.Size()) t.seqWithData[seq] = packet.Ack message.DataSeq = seq message.complete = false diff --git a/settings.go b/settings.go index 702dcfc..dd0675c 100644 --- a/settings.go +++ b/settings.go @@ -34,6 +34,7 @@ type AppSettings struct { inputDummy MultiOption outputDummy MultiOption outputStdout bool + outputNull bool inputTCP MultiOption outputTCP MultiOption @@ -81,6 +82,8 @@ func init() { flag.BoolVar(&Settings.outputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.") + flag.BoolVar(&Settings.outputNull, "output-null", false, "Used for testing inputs. Drops all requests.") + flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com") flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020") flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.") From 1ed8691b38016648870896d2359703db4113febb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 5 Jul 2016 17:09:56 +0300 Subject: [PATCH 21/79] Fix loopback for non local IP's --- raw_socket_listener/listener.go | 74 ++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index f639254..d225762 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -254,6 +254,28 @@ func (e *DeviceNotFoundError) Error() string { return msg } +func isLoopback(device pcap.Interface) bool { + if len(device.Addresses) == 0 { + return false + } + + switch device.Addresses[0].IP.String() { + case "127.0.0.1", "::1": + return true + } + + return false +} + +func listenAllInterfaces(addr string) bool { + switch addr { + case "", "0.0.0.0", "[::]", "::": + return true + default: + return false + } +} + func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) { devices, err := pcap.FindAllDevs() if err != nil { @@ -261,7 +283,7 @@ func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) { } for _, device := range devices { - if (addr == "" || addr == "0.0.0.0" || addr == "[::]" || addr == "::") && len(device.Addresses) > 0 { + if listenAllInterfaces(addr) && len(device.Addresses) > 0 || isLoopback(device) { interfaces = append(interfaces, device) continue } @@ -309,12 +331,26 @@ func (t *Listener) readPcap() { t.pcapHandles = append(t.pcapHandles, handle) var bpfDstHost, bpfSrcHost string - for i, addr := range device.Addresses { - bpfDstHost += "dst host " + addr.IP.String() - bpfSrcHost += "src host " + addr.IP.String() - if i != len(device.Addresses)-1 { - bpfDstHost += " or " - bpfSrcHost += " or " + var loopback = isLoopback(device) + + if loopback { + var allAddr []string + for _, dc := range devices { + for _, addr := range dc.Addresses { + allAddr = append(allAddr, "(dst host " + addr.IP.String() + " and src host " + addr.IP.String() + ")") + } + } + + bpfDstHost = strings.Join(allAddr, " or ") + bpfSrcHost = bpfDstHost + } else { + for i, addr := range device.Addresses { + bpfDstHost += "dst host " + addr.IP.String() + bpfSrcHost += "src host " + addr.IP.String() + if i != len(device.Addresses)-1 { + bpfDstHost += " or " + bpfSrcHost += " or " + } } } @@ -439,10 +475,26 @@ func (t *Listener) readPcap() { } addrMatched := false - for _, a := range device.Addresses { - if a.IP.Equal(net.IP(addrCheck)) { - addrMatched = true - break + + if loopback { + for _, dc := range devices { + if addrMatched { + break + } + for _, a := range dc.Addresses { + if a.IP.Equal(net.IP(addrCheck)) { + addrMatched = true + break + } + } + } + addrMatched = true + } else { + for _, a := range device.Addresses { + if a.IP.Equal(net.IP(addrCheck)) { + addrMatched = true + break + } } } From c128d4608855ae7e9ab0ffba9f143fc394f16322 Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Wed, 6 Jul 2016 14:10:02 -0400 Subject: [PATCH 22/79] calculate TCP payload slice sizes for Ethernet II packets less than 64 octets (#326) * remove unused bCLRF var. * calculate TCP payload slice sizes for Ethernet II packets less than 64 octets * simplify slicing off padding for Ethernet link packets less than 61 octets in length. * Make IPv4 header stripping comments more accurate. --- raw_socket_listener/listener.go | 7 ++++++- raw_socket_listener/tcp_message.go | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index d225762..b11b011 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -433,7 +433,12 @@ func (t *Listener) readPcap() { srcIP = data[12:16] dstIP = data[16:20] - data = data[ihl*4:] + // Stripping off the IP header + if len(packet.Data()) <= 60 && decoder == layers.LinkTypeEthernet { // Small Ethernet packets have padding + data = data[ihl * 4: int(binary.BigEndian.Uint16(data[2:4]))] + } else { + data = data[ihl * 4:] + } } else { // Truncated IP info if len(data) < 40 { diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 0f78fd8..15837bd 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -171,7 +171,6 @@ func (t *TCPMessage) checkSeqIntegrity() { t.seqMissing = false } -var bCLRF = []byte("\r\n") var bEmptyLine = []byte("\r\n\r\n") var bChunkEnd = []byte("0\r\n\r\n") From e65035dd951d1f3c2d188673d30bb5dbe45438fe Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Thu, 7 Jul 2016 09:30:51 -0400 Subject: [PATCH 23/79] Update elasticsearch output to use vendored elastigo and fix #331. (#333) * add elastigo submodule * add gou submodule * add go-hostpool submodule * use vendored elasticgo * update ReqUrl to ReqURL --- .gitmodules | 9 +++ elasticsearch.go | 103 ++++++++++++++------------- vendor/github.com/araddon/gou | 1 + vendor/github.com/bitly/go-hostpool | 1 + vendor/github.com/mattbaird/elastigo | 1 + 5 files changed, 64 insertions(+), 51 deletions(-) create mode 100644 .gitmodules create mode 160000 vendor/github.com/araddon/gou create mode 160000 vendor/github.com/bitly/go-hostpool create mode 160000 vendor/github.com/mattbaird/elastigo diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fef1eaa --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "vendor/github.com/mattbaird/elastigo"] + path = vendor/github.com/mattbaird/elastigo + url = https://github.com/mattbaird/elastigo +[submodule "vendor/github.com/araddon/gou"] + path = vendor/github.com/araddon/gou + url = https://github.com/araddon/gou +[submodule "vendor/github.com/bitly/go-hostpool"] + path = vendor/github.com/bitly/go-hostpool + url = https://github.com/bitly/go-hostpool diff --git a/elasticsearch.go b/elasticsearch.go index a930ac3..2ff4ab9 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -2,8 +2,7 @@ package main import ( "encoding/json" - "github.com/buger/elastigo/api" - "github.com/buger/elastigo/core" + "github.com/mattbaird/elastigo/lib" "github.com/buger/gor/proto" "log" "regexp" @@ -19,33 +18,34 @@ func (e *ESUriErorr) Error() string { type ESPlugin struct { Active bool ApiPort string + eConn *elastigo.Conn Host string Index string - indexor *core.BulkIndexer + indexor *elastigo.BulkIndexer done chan bool } type ESRequestResponse struct { - ReqUrl []byte `json:"Req_URL"` - ReqMethod []byte `json:"Req_Method"` - ReqUserAgent []byte `json:"Req_User-Agent"` - ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"` - ReqAccept []byte `json:"Req_Accept,omitempty"` - ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"` - ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"` - ReqConnection []byte `json:"Req_Connection,omitempty"` - ReqCookies []byte `json:"Req_Cookies,omitempty"` - RespStatus []byte `json:"Resp_Status"` - RespStatusCode []byte `json:"Resp_Status-Code"` - RespProto []byte `json:"Resp_Proto,omitempty"` - RespContentLength []byte `json:"Resp_Content-Length,omitempty"` - RespContentType []byte `json:"Resp_Content-Type,omitempty"` - RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"` - RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"` - RespExpires []byte `json:"Resp_Expires,omitempty"` - RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"` - RespVary []byte `json:"Resp_Vary,omitempty"` - RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"` + ReqURL string `json:"Req_URL"` + ReqMethod string `json:"Req_Method"` + ReqUserAgent string `json:"Req_User-Agent"` + ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"` + ReqAccept string `json:"Req_Accept,omitempty"` + ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"` + ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"` + ReqConnection string `json:"Req_Connection,omitempty"` + ReqCookies string `json:"Req_Cookies,omitempty"` + RespStatus string `json:"Resp_Status"` + RespStatusCode string `json:"Resp_Status-Code"` + RespProto string `json:"Resp_Proto,omitempty"` + RespContentLength string `json:"Resp_Content-Length,omitempty"` + RespContentType string `json:"Resp_Content-Type,omitempty"` + RespTransferEncoding string `json:"Resp_Transfer-Encoding,omitempty"` + RespContentEncoding string `json:"Resp_Content-Encoding,omitempty"` + RespExpires string `json:"Resp_Expires,omitempty"` + RespCacheControl string `json:"Resp_Cache-Control,omitempty"` + RespVary string `json:"Resp_Vary,omitempty"` + RespSetCookie string `json:"Resp_Set-Cookie,omitempty"` Rtt int64 `json:"RTT"` Timestamp time.Time } @@ -76,24 +76,24 @@ func (p *ESPlugin) Init(URI string) { if err != nil { log.Fatal("Can't initialize ElasticSearch plugin.", err) } + p.eConn = elastigo.NewConn() + p.eConn.SetPort(p.ApiPort) + p.eConn.SetHosts([]string{p.Host}) - api.Domain = p.Host - api.Port = p.ApiPort - - p.indexor = core.NewBulkIndexerErrors(50, 60) + p.indexor = p.eConn.NewBulkIndexerErrors(50, 60) p.done = make(chan bool) - p.indexor.Run(p.done) + p.indexor.Start() // Only start the ErrorHandler goroutine when in verbose mode // no need to burn ressources otherwise - // go p.ErrorHandler() + go p.ErrorHandler() log.Println("Initialized Elasticsearch Plugin") return } func (p *ESPlugin) IndexerShutdown() { - p.done <- true + p.indexor.Stop() return } @@ -118,28 +118,29 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) { } t := time.Now() rtt := p.RttDurationToMs(stop.Sub(start)) + req = payloadBody(req) esResp := ESRequestResponse{ - ReqUrl: proto.Path(req), - ReqMethod: proto.Method(req), - ReqUserAgent: proto.Header(req, []byte("User-Agent")), - ReqAcceptLanguage: proto.Header(req, []byte("Accept-Language")), - ReqAccept: proto.Header(req, []byte("Accept")), - ReqAcceptEncoding: proto.Header(req, []byte("Accept-Encoding")), - ReqIfModifiedSince: proto.Header(req, []byte("If-Modified-Since")), - ReqConnection: proto.Header(req, []byte("Connection")), - ReqCookies: proto.Header(req, []byte("Cookie")), - RespStatus: proto.Status(resp), - RespStatusCode: proto.Status(resp), - RespProto: proto.Method(resp), - RespContentLength: proto.Header(resp, []byte("Content-Length")), - RespContentType: proto.Header(resp, []byte("Content-Type")), - RespTransferEncoding: proto.Header(resp, []byte("Transfer-Encoding")), - RespContentEncoding: proto.Header(resp, []byte("Content-Encoding")), - RespExpires: proto.Header(resp, []byte("Expires")), - RespCacheControl: proto.Header(resp, []byte("Cache-Control")), - RespVary: proto.Header(resp, []byte("Vary")), - RespSetCookie: proto.Header(resp, []byte("Set-Cookie")), + ReqURL: string(proto.Path(req)), + ReqMethod: string(proto.Method(req)), + ReqUserAgent: string(proto.Header(req, []byte("User-Agent"))), + ReqAcceptLanguage: string(proto.Header(req, []byte("Accept-Language"))), + ReqAccept: string(proto.Header(req, []byte("Accept"))), + ReqAcceptEncoding: string(proto.Header(req, []byte("Accept-Encoding"))), + ReqIfModifiedSince: string(proto.Header(req, []byte("If-Modified-Since"))), + ReqConnection: string(proto.Header(req, []byte("Connection"))), + ReqCookies: string(proto.Header(req, []byte("Cookie"))), + RespStatus: string(proto.Status(resp)), + RespStatusCode: string(proto.Status(resp)), + RespProto: string(proto.Method(resp)), + RespContentLength: string(proto.Header(resp, []byte("Content-Length"))), + RespContentType: string(proto.Header(resp, []byte("Content-Type"))), + RespTransferEncoding: string(proto.Header(resp, []byte("Transfer-Encoding"))), + RespContentEncoding: string(proto.Header(resp, []byte("Content-Encoding"))), + RespExpires: string(proto.Header(resp, []byte("Expires"))), + RespCacheControl: string(proto.Header(resp, []byte("Cache-Control"))), + RespVary: string(proto.Header(resp, []byte("Vary"))), + RespSetCookie: string(proto.Header(resp, []byte("Set-Cookie"))), Rtt: rtt, Timestamp: t, } @@ -147,7 +148,7 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) { if err != nil { log.Println(err) } else { - p.indexor.Index(p.Index, "RequestResponse", "", "", &t, j, true) + p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j) } return } diff --git a/vendor/github.com/araddon/gou b/vendor/github.com/araddon/gou new file mode 160000 index 0000000..50a94aa --- /dev/null +++ b/vendor/github.com/araddon/gou @@ -0,0 +1 @@ +Subproject commit 50a94aa4a3fb69e8fbde05df290fcb49fa685e07 diff --git a/vendor/github.com/bitly/go-hostpool b/vendor/github.com/bitly/go-hostpool new file mode 160000 index 0000000..d0e59c2 --- /dev/null +++ b/vendor/github.com/bitly/go-hostpool @@ -0,0 +1 @@ +Subproject commit d0e59c22a56e8dadfed24f74f452cea5a52722d2 diff --git a/vendor/github.com/mattbaird/elastigo b/vendor/github.com/mattbaird/elastigo new file mode 160000 index 0000000..34c4c4d --- /dev/null +++ b/vendor/github.com/mattbaird/elastigo @@ -0,0 +1 @@ +Subproject commit 34c4c4d8425cbdcbc8e257943a2044d5e9f7dab5 From a5cc943af4bb7b7e5da80e199d6e50420b773074 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 7 Jul 2016 18:11:33 +0300 Subject: [PATCH 24/79] Fix padded packet check --- raw_socket_listener/listener.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index b11b011..fc2be20 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -422,6 +422,7 @@ func (t *Listener) readPcap() { data = packet.Data()[of:] version := uint8(data[0]) >> 4 + ipLength := int(binary.BigEndian.Uint16(data[2:4])) if version == 4 { ihl := uint8(data[0]) & 0x0F @@ -433,12 +434,25 @@ func (t *Listener) readPcap() { srcIP = data[12:16] dstIP = data[16:20] - // Stripping off the IP header - if len(packet.Data()) <= 60 && decoder == layers.LinkTypeEthernet { // Small Ethernet packets have padding - data = data[ihl * 4: int(binary.BigEndian.Uint16(data[2:4]))] - } else { - data = data[ihl * 4:] + + // Too small IP packet + if ipLength < 20 { + continue } + + // Invalid length + if ihl * 4 > ipLength { + continue + } + + if cmp := len(data) - ipLength; cmp > 0 { + data = data[:ipLength] + } else if cmp < 0 { + // Truncated packet + continue + } + + data = data[ihl * 4:] } else { // Truncated IP info if len(data) < 40 { From 12287d8baeff73e4a84d672035b3a80f6faf048c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 7 Jul 2016 18:41:29 +0300 Subject: [PATCH 25/79] Fix type check, and add asserts submodule --- .gitmodules | 3 +++ raw_socket_listener/listener.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index fef1eaa..1c1686b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "vendor/github.com/bitly/go-hostpool"] path = vendor/github.com/bitly/go-hostpool url = https://github.com/bitly/go-hostpool +[submodule "vendor/github.com/bmizerany/assert"] + path = vendor/github.com/bmizerany/assert + url = https://github.com/bmizerany/assert diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index fc2be20..a4c88fd 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -441,7 +441,7 @@ func (t *Listener) readPcap() { } // Invalid length - if ihl * 4 > ipLength { + if int(ihl * 4) > ipLength { continue } From c78933f1df9b2ef1636c2de101f314478a3bd6ef Mon Sep 17 00:00:00 2001 From: Or Tzabary Date: Fri, 8 Jul 2016 17:05:46 +0300 Subject: [PATCH 26/79] Feature flag --- gor.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/gor.go b/gor.go index 8788a74..c0f06c7 100644 --- a/gor.go +++ b/gor.go @@ -22,6 +22,7 @@ var ( mode string cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") memprofile = flag.String("memprofile", "", "write memory profile to this file") + timeout = flag.Int("timeout", 0, "timeout to stop gor, value in seconds") ) func loggingMiddleware(next http.Handler) http.Handler { @@ -88,7 +89,15 @@ func main() { os.Exit(1) }() - Start(nil) + if *timeout >= 1 { + log.Println("Running gor with timeout of", *timeout, "seconds") + stop := make(chan int) + timeoutGor(stop, *timeout) + + Start(stop) + } else { + Start(nil) + } } func profileCPU(cpuprofile string) { @@ -119,3 +128,10 @@ func profileMEM(memprofile string) { }) } } + +func timeoutGor(stop chan int, seconds int) { + time.AfterFunc(time.Duration(seconds)*time.Second, func() { + log.Println("Stopping gor after", seconds, "seconds") + close(stop) + }) +} From 6620d256264802aa4d67d26ad92437a758be4ff8 Mon Sep 17 00:00:00 2001 From: Or Tzabary Date: Fri, 8 Jul 2016 18:57:42 +0300 Subject: [PATCH 27/79] Change flag name, use of DurationVar instead of int --- gor.go | 13 ++++++------- settings.go | 8 +++++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/gor.go b/gor.go index c0f06c7..70f3fa7 100644 --- a/gor.go +++ b/gor.go @@ -22,7 +22,6 @@ var ( mode string cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") memprofile = flag.String("memprofile", "", "write memory profile to this file") - timeout = flag.Int("timeout", 0, "timeout to stop gor, value in seconds") ) func loggingMiddleware(next http.Handler) http.Handler { @@ -89,10 +88,10 @@ func main() { os.Exit(1) }() - if *timeout >= 1 { - log.Println("Running gor with timeout of", *timeout, "seconds") + if Settings.exitAfter >= 1 { + log.Println("Running gor for a duration of", Settings.exitAfter) stop := make(chan int) - timeoutGor(stop, *timeout) + stopAfter(stop, Settings.exitAfter) Start(stop) } else { @@ -129,9 +128,9 @@ func profileMEM(memprofile string) { } } -func timeoutGor(stop chan int, seconds int) { - time.AfterFunc(time.Duration(seconds)*time.Second, func() { - log.Println("Stopping gor after", seconds, "seconds") +func stopAfter(stop chan int, exitAfter time.Duration) { + time.AfterFunc(exitAfter, func() { + log.Println("Stopping gor, duration of", exitAfter, "reached") close(stop) }) } diff --git a/settings.go b/settings.go index dd0675c..1b6104e 100644 --- a/settings.go +++ b/settings.go @@ -25,9 +25,10 @@ func (h *MultiOption) Set(value string) error { // AppSettings is the struct of main configuration type AppSettings struct { - verbose bool - debug bool - stats bool + verbose bool + debug bool + stats bool + exitAfter time.Duration splitOutput bool @@ -74,6 +75,7 @@ func init() { flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output") flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag") flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output") + flag.DurationVar(&Settings.exitAfter, "exit-after", 0, "exit after specified duration") flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.") From dae6a8915de74aaa08f6d697d20037cdb353f804 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 8 Jul 2016 19:27:20 +0300 Subject: [PATCH 28/79] Refactor exit-after --- emitter.go | 8 +------- gor.go | 37 ++++++++++++++++++------------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/emitter.go b/emitter.go index 9575484..1fc2a83 100644 --- a/emitter.go +++ b/emitter.go @@ -32,13 +32,7 @@ func Start(stop chan int) { for { select { case <-stop: - pluginMu.Lock() - for _, p := range Plugins.All { - if cp, ok := p.(io.Closer); ok { - cp.Close() - } - } - pluginMu.Unlock() + finalize() return case <-time.After(100 * time.Millisecond): } diff --git a/gor.go b/gor.go index 70f3fa7..085eb98 100644 --- a/gor.go +++ b/gor.go @@ -78,27 +78,33 @@ func main() { signal.Notify(c, os.Interrupt, syscall.SIGTERM) go func() { <-c - - for _, p := range Plugins.All { - if cp, ok := p.(io.Closer); ok { - cp.Close() - } - } - + finalize() os.Exit(1) }() - if Settings.exitAfter >= 1 { + if Settings.exitAfter > 0 { log.Println("Running gor for a duration of", Settings.exitAfter) - stop := make(chan int) - stopAfter(stop, Settings.exitAfter) + closeCh := make(chan int) - Start(stop) + time.AfterFunc(Settings.exitAfter, func() { + log.Println("Stopping gor after", Settings.exitAfter) + close(closeCh) + }) + + Start(closeCh) } else { Start(nil) } } +func finalize() { + for _, p := range Plugins.All { + if cp, ok := p.(io.Closer); ok { + cp.Close() + } + } +} + func profileCPU(cpuprofile string) { if cpuprofile != "" { f, err := os.Create(cpuprofile) @@ -126,11 +132,4 @@ func profileMEM(memprofile string) { f.Close() }) } -} - -func stopAfter(stop chan int, exitAfter time.Duration) { - time.AfterFunc(exitAfter, func() { - log.Println("Stopping gor, duration of", exitAfter, "reached") - close(stop) - }) -} +} \ No newline at end of file From 8e0a8b31a976594ab3322a18425dd77c394933b8 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 8 Jul 2016 19:38:45 +0300 Subject: [PATCH 29/79] Add missing submodule --- vendor/github.com/bmizerany/assert | 1 + 1 file changed, 1 insertion(+) create mode 160000 vendor/github.com/bmizerany/assert diff --git a/vendor/github.com/bmizerany/assert b/vendor/github.com/bmizerany/assert new file mode 160000 index 0000000..b7ed37b --- /dev/null +++ b/vendor/github.com/bmizerany/assert @@ -0,0 +1 @@ +Subproject commit b7ed37b82869576c289d7d97fb2bbd8b64a0cb28 From 24fc5b00785b777d1dc7c8749fc5fd279426685b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 10 Jul 2016 19:13:57 +0300 Subject: [PATCH 30/79] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a161984..1d0af52 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ Here is basic workflow: The listener server catches http traffic and sends it to ![Diagram](http://i.imgur.com/9mqj2SK.png) +Check [latest documentation](http://github.com/buger/gor/wiki). + ## Installation Download latest binary from https://github.com/buger/gor/releases or [compile by yourself](https://github.com/buger/gor/wiki/Compilation). @@ -22,13 +24,14 @@ The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` whic If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`. See the our wiki and especially [Getting started](https://github.com/buger/gor/wiki/Getting-Started) wiki page for more info. +## Newsletter +Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project. ## Want to Upgrade? I also sell Gor Pro, extensions to Gor which provide more features, a commercial-friendly license and allow you to support high quality open source development all at the same time. Please see the Gor [homepage](https://gortool.com/) for more detail. -Subscribe to the [quarterly newsletter](https://tinyletter.com/gor) to stay informed about the latest features and changes to Gor and its bigger siblings. ## Problems? If you have a problem, please review the [FAQ](https://github.com/buger/gor/wiki/FAQ) and [Troubleshooting](https://github.com/buger/gor/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/gor/issues) for your problem is also a good idea. From 35bb655f2d97fbb9301d94ab74900d6324aec126 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 13 Jul 2016 15:59:01 +0300 Subject: [PATCH 31/79] Fix response meta --- protocol.go | 1 + 1 file changed, 1 insertion(+) diff --git a/protocol.go b/protocol.go index 50d6943..63c4560 100644 --- a/protocol.go +++ b/protocol.go @@ -69,6 +69,7 @@ func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) ( copy(header[3+len(uuid):], sTime) if latency != -1 { + header[3+len(uuid)+len(sTime)] = ' ' copy(header[4+len(uuid)+len(sTime):], sLatency) } From a6d263df1c72b276971c012ebcf15d090dcb1b63 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 13 Jul 2016 16:01:25 +0300 Subject: [PATCH 32/79] Add support for binary input/output --- Dockerfile | 4 +- Makefile | 2 +- input_raw.go | 18 ++- input_raw_test.go | 12 +- middleware_test.go | 4 +- output_binary.go | 175 ++++++++++++++++++++++ output_http.go | 2 +- plugins.go | 6 +- protocol.go | 1 + raw_socket_listener/listener.go | 49 +++--- raw_socket_listener/listener_test.go | 16 +- raw_socket_listener/tcp_message.go | 73 +++++---- raw_socket_listener/tcp_message_test.go | 2 +- settings.go | 24 ++- tcp_client.go | 190 ++++++++++++++++++++++++ 15 files changed, 494 insertions(+), 84 deletions(-) create mode 100644 output_binary.go create mode 100644 tcp_client.go diff --git a/Dockerfile b/Dockerfile index 98fb7a4..91435a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,8 @@ RUN wget http://www.tcpdump.org/release/libpcap-1.7.4.tar.gz && tar xzf libpcap- RUN go get github.com/google/gopacket RUN go get -u github.com/golang/lint/golint -WORKDIR /go/src/github.com/buger/gor/ -ADD . /go/src/github.com/buger/gor/ +WORKDIR /go/src/github.com/buger/gor-pro/ +ADD . /go/src/github.com/buger/gor-pro/ RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java RUN go get \ No newline at end of file diff --git a/Makefile b/Makefile index c0c1e45..f7807d8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_null.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_null.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go tcp_client.go output_binary.go SOURCE_PATH = /go/src/github.com/buger/gor/ PORT = 8000 FADDR = :8000 diff --git a/input_raw.go b/input_raw.go index 0236a2c..f733670 100644 --- a/input_raw.go +++ b/input_raw.go @@ -1,8 +1,8 @@ package main import ( - "github.com/buger/gor/proto" - raw "github.com/buger/gor/raw_socket_listener" + "github.com/buger/gor-pro/proto" + raw "github.com/buger/gor-pro/raw_socket_listener" "log" "net" "time" @@ -18,6 +18,7 @@ type RAWInput struct { realIPHeader []byte trackResponse bool listener *raw.Listener + protocol raw.TCPProtocol } // Available engines for intercepting traffic @@ -28,7 +29,7 @@ const ( ) // NewRAWInput constructor for RAWInput. Accepts address with port as argument. -func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string) (i *RAWInput) { +func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, protocol string) (i *RAWInput) { i = new(RAWInput) i.data = make(chan *raw.TCPMessage) i.address = address @@ -38,6 +39,15 @@ func NewRAWInput(address string, engine int, trackResponse bool, expire time.Dur i.quit = make(chan bool) i.trackResponse = trackResponse + switch protocol { + case "http": + i.protocol = raw.ProtocolHTTP + case "binary": + i.protocol = raw.ProtocolBinary + default: + log.Fatal("Unsupported protocol:", protocol) + } + i.listen(address) i.listener.IsReady() @@ -80,7 +90,7 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire) + i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol) ch := i.listener.Receiver() diff --git a/input_raw_test.go b/input_raw_test.go index b8fe1b1..b44e37e 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -43,7 +43,7 @@ func TestRAWInputIPv4(t *testing.T) { var respCounter, reqCounter int64 - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP", "http") defer input.Close() output := NewTestOutput(func(data []byte) { @@ -102,7 +102,7 @@ func TestRAWInputIPv6(t *testing.T) { var respCounter, reqCounter int64 - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http") defer input.Close() output := NewTestOutput(func(data []byte) { @@ -153,7 +153,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "") + input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http") defer input.Close() // We will use it to get content of raw HTTP request @@ -216,7 +216,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "") + input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http") defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -280,7 +280,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http") defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -325,7 +325,7 @@ func BenchmarkRAWInput(b *testing.B) { var respCounter, reqCounter int64 - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http") defer input.Close() output := NewTestOutput(func(data []byte) { diff --git a/middleware_test.go b/middleware_test.go index beaed15..46f9b28 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -117,7 +117,7 @@ func TestEchoMiddleware(t *testing.T) { // Catch traffic from one service fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "") + input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http") defer input.Close() // And redirect to another @@ -179,7 +179,7 @@ func TestTokenMiddleware(t *testing.T) { fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) // Catch traffic from one service - input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "") + input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http") defer input.Close() // And redirect to another diff --git a/output_binary.go b/output_binary.go new file mode 100644 index 0000000..6134e65 --- /dev/null +++ b/output_binary.go @@ -0,0 +1,175 @@ +package main + +import ( + "io" + "sync/atomic" + "time" +) + +// BinaryOutputConfig struct for holding binary output configuration +type BinaryOutputConfig struct { + workers int + Timeout time.Duration + BufferSize int + Debug bool + TrackResponses bool +} + +// BinaryOutput plugin manage pool of workers which send request to replayed server +// By default workers pool is dynamic and starts with 10 workers +// You can specify fixed number of workers using `--output-tcp-workers` +type BinaryOutput struct { + // Keep this as first element of struct because it guarantees 64bit + // alignment. atomic.* functions crash on 32bit machines if operand is not + // aligned at 64bit. See https://github.com/golang/go/issues/599 + activeWorkers int64 + + address string + queue chan []byte + + responses chan response + + needWorker chan int + + config *BinaryOutputConfig + + queueStats *GorStat +} + +// NewBinaryOutput constructor for BinaryOutput +// Initialize workers +func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer { + o := new(BinaryOutput) + + o.address = address + o.config = config + + o.queue = make(chan []byte, 1000) + o.responses = make(chan response, 1000) + o.needWorker = make(chan int, 1) + + // Initial workers count + if o.config.workers == 0 { + o.needWorker <- initialDynamicWorkers + } else { + o.needWorker <- o.config.workers + } + + if len(Settings.middleware) > 0 { + o.config.TrackResponses = true + } + + go o.workerMaster() + + return o +} + +func (o *BinaryOutput) workerMaster() { + for { + newWorkers := <-o.needWorker + for i := 0; i < newWorkers; i++ { + go o.startWorker() + } + + // Disable dynamic scaling if workers poll fixed size + if o.config.workers != 0 { + return + } + } +} + +func (o *BinaryOutput) startWorker() { + client := NewTCPClient(o.address, &TCPClientConfig{ + Debug: o.config.Debug, + Timeout: o.config.Timeout, + ResponseBufferSize: o.config.BufferSize, + }) + + deathCount := 0 + + atomic.AddInt64(&o.activeWorkers, 1) + + for { + select { + case data := <-o.queue: + o.sendRequest(client, data) + deathCount = 0 + case <-time.After(time.Millisecond * 100): + // When dynamic scaling enabled workers die after 2s of inactivity + if o.config.workers == 0 { + deathCount++ + } else { + continue + } + + if deathCount > 20 { + workersCount := atomic.LoadInt64(&o.activeWorkers) + + // At least 1 startWorker should be alive + if workersCount != 1 { + atomic.AddInt64(&o.activeWorkers, -1) + return + } + } + } + } +} + +func (o *BinaryOutput) Write(data []byte) (n int, err error) { + if !isRequestPayload(data) { + return len(data), nil + } + + buf := make([]byte, len(data)) + copy(buf, data) + + o.queue <- buf + + if o.config.workers == 0 { + workersCount := atomic.LoadInt64(&o.activeWorkers) + + if len(o.queue) > int(workersCount) { + o.needWorker <- len(o.queue) + } + } + + return len(data), nil +} + +func (o *BinaryOutput) Read(data []byte) (int, error) { + resp := <-o.responses + + Debug("[OUTPUT-TCP] Received response:", string(resp.payload)) + + header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime) + copy(data[0:len(header)], header) + copy(data[len(header):], resp.payload) + + return len(resp.payload) + len(header), nil +} + +func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) { + meta := payloadMeta(request) + if len(meta) < 2 { + return + } + uuid := meta[1] + + body := payloadBody(request) + + start := time.Now() + resp, err := client.Send(body) + stop := time.Now() + + if err != nil { + Debug("Request error:", err) + } + + if o.config.TrackResponses { + o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()} + } +} + +func (o *BinaryOutput) String() string { + return "TCP output: " + o.address +} diff --git a/output_http.go b/output_http.go index d442b83..397c96c 100644 --- a/output_http.go +++ b/output_http.go @@ -5,7 +5,7 @@ import ( "sync/atomic" "time" - "github.com/buger/gor/proto" + "github.com/buger/gor-pro/proto" ) const initialDynamicWorkers = 10 diff --git a/plugins.go b/plugins.go index 1a0f6db..42e5699 100644 --- a/plugins.go +++ b/plugins.go @@ -107,7 +107,7 @@ func InitPlugins() { } for _, options := range Settings.inputRAW { - registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, time.Duration(0), Settings.inputRAWRealIPHeader) + registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, time.Duration(0), Settings.inputRAWRealIPHeader, Settings.inputRAWProtocol) } for _, options := range Settings.inputTCP { @@ -142,4 +142,8 @@ func InitPlugins() { for _, options := range Settings.outputHTTP { registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig) } + + for _, options := range Settings.outputBinary { + registerPlugin(NewBinaryOutput, options, &Settings.outputBinaryConfig) + } } diff --git a/protocol.go b/protocol.go index 50d6943..63c4560 100644 --- a/protocol.go +++ b/protocol.go @@ -69,6 +69,7 @@ func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) ( copy(header[3+len(uuid):], sTime) if latency != -1 { + header[3+len(uuid)+len(sTime)] = ' ' copy(header[4+len(uuid)+len(sTime):], sLatency) } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index a4c88fd..4d21cf9 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -16,7 +16,7 @@ import ( "bytes" "encoding/binary" "fmt" - "github.com/buger/gor/proto" + "github.com/buger/gor-pro/proto" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/google/gopacket/pcap" @@ -69,6 +69,8 @@ type Listener struct { quit chan bool readyCh chan bool + + protocol TCPProtocol } type request struct { @@ -85,7 +87,7 @@ const ( ) // NewListener creates and initializes new Listener object -func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration) (l *Listener) { +func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, protocol TCPProtocol) (l *Listener) { l = &Listener{} l.packetsChan = make(chan []byte, 10000) @@ -99,6 +101,7 @@ func NewListener(addr string, port string, engine int, trackResponse bool, expir l.respAliases = make(map[uint32]*TCPMessage) l.respWithoutReq = make(map[uint32]tcpID) l.trackResponse = trackResponse + l.protocol = protocol l.addr = addr _port, _ := strconv.Atoi(port) @@ -176,7 +179,7 @@ func (t *Listener) dispatchMessage(message *TCPMessage) { t.deleteMessage(message) - if !message.complete { + if t.protocol == ProtocolHTTP && !message.complete { if !message.IsIncoming { delete(t.respAliases, message.Ack) delete(t.respWithoutReq, message.Ack) @@ -667,28 +670,30 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { isIncoming := packet.DestPort == t.port - // Seek for 100-expect chunks - if parentAck, ok := t.seqWithData[packet.Seq]; ok { - // In case if non-first data chunks comes first - for _, m := range t.messages { - if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) { - t.deleteMessage(m) + if t.protocol == ProtocolHTTP { + // Seek for 100-expect chunks + if parentAck, ok := t.seqWithData[packet.Seq]; ok { + // In case if non-first data chunks comes first + for _, m := range t.messages { + if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) { + t.deleteMessage(m) - if m.AssocMessage != nil { - m.setAssocMessage(nil) - } + if m.AssocMessage != nil { + m.setAssocMessage(nil) + } - for _, pkt := range m.packets { - // log.Println("Updating ack", parentAck, pkt.Ack) - pkt.UpdateAck(parentAck) - // Re-queue this packets - t.processTCPPacket(pkt) + for _, pkt := range m.packets { + // log.Println("Updating ack", parentAck, pkt.Ack) + pkt.UpdateAck(parentAck) + // Re-queue this packets + t.processTCPPacket(pkt) + } } } - } - t.ackAliases[packet.Ack] = parentAck - packet.UpdateAck(parentAck) + t.ackAliases[packet.Ack] = parentAck + packet.UpdateAck(parentAck) + } } if alias, ok := t.ackAliases[packet.Ack]; ok { @@ -704,7 +709,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { message, ok := t.messages[packet.ID] if !ok { - message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming) + message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol) t.messages[packet.ID] = message if !isIncoming { @@ -721,7 +726,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { message.AddPacket(packet) // Handling Expect: 100-continue requests - if message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 { + if t.protocol == ProtocolHTTP && message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 { seq := packet.Seq + uint32(message.Size()) t.seqWithData[seq] = packet.Ack message.DataSeq = seq diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index b66b23a..eee85dd 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -12,7 +12,7 @@ import ( func TestRawListenerInput(t *testing.T) { var req, resp *TCPMessage - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) @@ -49,7 +49,7 @@ func TestRawListenerInput(t *testing.T) { func TestRawListenerInputWithoutResponse(t *testing.T) { var req *TCPMessage - listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond) + listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) @@ -71,7 +71,7 @@ func TestRawListenerInputWithoutResponse(t *testing.T) { func TestRawListenerResponse(t *testing.T) { var req, resp *TCPMessage - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) @@ -109,7 +109,7 @@ func TestRawListenerResponse(t *testing.T) { } func TestShort100Continue(t *testing.T) { - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n")) @@ -129,7 +129,7 @@ func TestShort100Continue(t *testing.T) { // Response comes before Request func Test100ContinueWrongOrder(t *testing.T) { - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n")) @@ -148,7 +148,7 @@ func Test100ContinueWrongOrder(t *testing.T) { } func TestAlt100ContinueHeaderOrder(t *testing.T) { - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n")) @@ -309,7 +309,7 @@ func permutation(n int, list []*TCPPacket) []*TCPPacket { // Response comes before Request func TestRawListenerChunkedWrongOrder(t *testing.T) { - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n")) @@ -387,7 +387,7 @@ func getMessage() []*TCPPacket { // Response comes before Request func TestRawListenerBench(t *testing.T) { - l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond) + l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, ProtocolHTTP) defer l.Close() // Should re-construct message from all possible combinations diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 15837bd..ba9388d 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -14,6 +14,13 @@ import ( var _ = log.Println +type TCPProtocol uint8 + +const ( + ProtocolHTTP TCPProtocol = 0 + ProtocolBinary TCPProtocol = 1 +) + // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence // Its needed because all TCP message can be fragmented or re-transmitted // @@ -37,6 +44,8 @@ type TCPMessage struct { delChan chan *TCPMessage + protocol TCPProtocol + /* HTTP specific variables */ methodType httpMethodType bodyType httpBodyType @@ -48,8 +57,8 @@ type TCPMessage struct { } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(Seq, Ack uint32, IsIncoming bool) (msg *TCPMessage) { - msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming} +func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, protocol TCPProtocol) (msg *TCPMessage) { + msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, protocol: protocol} msg.Start = time.Now() return @@ -95,48 +104,46 @@ func (t *TCPMessage) Size() (size int) { // AddPacket to the message and ensure packet uniqueness // TCP allows that packet can be re-send multiple times func (t *TCPMessage) AddPacket(packet *TCPPacket) { - packetFound := false - for _, pkt := range t.packets { if packet.Seq == pkt.Seq { - packetFound = true - break + return } } - if !packetFound { - // Packets not always captured in same Seq order, and sometimes we need to prepend - if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq { - t.packets = append(t.packets, packet) - } else if packet.Seq < t.packets[0].Seq { - t.packets = append([]*TCPPacket{packet}, t.packets...) - t.Seq = packet.Seq // Message Seq should indicated starting seq - } else { // insert somewhere in the middle... - for i, p := range t.packets { - if packet.Seq < p.Seq { - t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...) - break - } + // Packets not always captured in same Seq order, and sometimes we need to prepend + if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq { + t.packets = append(t.packets, packet) + } else if packet.Seq < t.packets[0].Seq { + t.packets = append([]*TCPPacket{packet}, t.packets...) + t.Seq = packet.Seq // Message Seq should indicated starting seq + } else { // insert somewhere in the middle... + for i, p := range t.packets { + if packet.Seq < p.Seq { + t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...) + break } } + } - if t.IsIncoming { - t.End = time.Now() - } else { - t.End = time.Now().Add(time.Millisecond) - } + if t.IsIncoming { + t.End = time.Now() + } else { + t.End = time.Now().Add(time.Millisecond) + } - if packet.OrigAck != 0 { - t.DataAck = packet.OrigAck - } + if packet.OrigAck != 0 { + t.DataAck = packet.OrigAck } t.checkSeqIntegrity() - t.updateHeadersPacket() - t.updateMethodType() - t.updateBodyType() - t.checkIfComplete() - t.check100Continue() + + if t.protocol == ProtocolHTTP { + t.updateHeadersPacket() + t.updateMethodType() + t.updateBodyType() + t.check100Continue() + t.checkIfComplete() + } } // Check if there is missing packet @@ -156,7 +163,7 @@ func (t *TCPMessage) checkSeqIntegrity() { nextSeq := p.Seq + uint32(len(p.Data)) if np.Seq != nextSeq { - if t.expectType == httpExpect100Continue { + if t.protocol == ProtocolHTTP && t.expectType == httpExpect100Continue { if np.Seq != nextSeq+22 { t.seqMissing = true return diff --git a/raw_socket_listener/tcp_message_test.go b/raw_socket_listener/tcp_message_test.go index 0af3bd9..618ea3f 100644 --- a/raw_socket_listener/tcp_message_test.go +++ b/raw_socket_listener/tcp_message_test.go @@ -36,7 +36,7 @@ func buildMessage(p *TCPPacket) *TCPMessage { isIncoming = true } - m := NewTCPMessage(p.Seq, p.Ack, isIncoming) + m := NewTCPMessage(p.Seq, p.Ack, isIncoming, ProtocolHTTP) m.AddPacket(p) return m diff --git a/settings.go b/settings.go index dd0675c..425c13d 100644 --- a/settings.go +++ b/settings.go @@ -49,13 +49,18 @@ type AppSettings struct { inputRAWEngine string inputRAWTrackResponse bool inputRAWRealIPHeader string + inputRAWProtocol string middleware string inputHTTP MultiOption - outputHTTP MultiOption + outputHTTP MultiOption outputHTTPConfig HTTPOutputConfig + + outputBinary MultiOption + outputBinaryConfig BinaryOutputConfig + modifierConfig HTTPModifierConfig } @@ -106,6 +111,8 @@ func init() { flag.StringVar(&Settings.inputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`") + flag.StringVar(&Settings.inputRAWProtocol, "input-raw-protocol", "http", "Specify application protocol of intercepted traffic. Possible values: http, binary") + flag.StringVar(&Settings.inputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP") flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command") @@ -113,16 +120,27 @@ func init() { flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") + + /* outputHTTPConfig */ flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") - flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.") flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.") - flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") + /* outputHTTPConfig */ + + + flag.Var(&Settings.outputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80") + /* outputBinaryConfig */ + flag.IntVar(&Settings.outputBinaryConfig.BufferSize, "output-tcp-response-buffer", 0, "TCP response buffer size, all data after this size will be discarded.") + flag.IntVar(&Settings.outputBinaryConfig.workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") + flag.DurationVar(&Settings.outputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s") + + flag.BoolVar(&Settings.outputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.") + /* outputBinaryConfig */ flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead") diff --git a/tcp_client.go b/tcp_client.go new file mode 100644 index 0000000..677c2d4 --- /dev/null +++ b/tcp_client.go @@ -0,0 +1,190 @@ +package main + +import ( + "crypto/tls" + "io" + "log" + "net" + "runtime/debug" + "syscall" + "time" +) + +type TCPClientConfig struct { + Debug bool + ConnectionTimeout time.Duration + Timeout time.Duration + ResponseBufferSize int + Secure bool +} + +type TCPClient struct { + baseURL string + addr string + conn net.Conn + respBuf []byte + config *TCPClientConfig + redirectsCount int +} + +func NewTCPClient(addr string, config *TCPClientConfig) *TCPClient { + if config.Timeout.Nanoseconds() == 0 { + config.Timeout = 5 * time.Second + } + + config.ConnectionTimeout = config.Timeout + + if config.ResponseBufferSize == 0 { + config.ResponseBufferSize = 100 * 1024 // 100kb + } + + client := &TCPClient{config: config, addr: addr} + client.respBuf = make([]byte, config.ResponseBufferSize) + + return client +} + +func (c *TCPClient) Connect() (err error) { + c.Disconnect() + + c.conn, err = net.DialTimeout("tcp", c.addr, c.config.ConnectionTimeout) + + if c.config.Secure { + tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) + + if err = tlsConn.Handshake(); err != nil { + return + } + + c.conn = tlsConn + } + + return +} + +func (c *TCPClient) Disconnect() { + if c.conn != nil { + c.conn.Close() + c.conn = nil + Debug("[TCPClient] Disconnected: ", c.baseURL) + } +} + +func (c *TCPClient) isAlive() bool { + one := make([]byte, 1) + + // Ready 1 byte from socket without timeout to check if it not closed + c.conn.SetReadDeadline(time.Now().Add(time.Millisecond)) + _, err := c.conn.Read(one) + + if err == nil { + return true + } else if err == io.EOF { + if c.config.Debug { + Debug("[TCPClient] connection closed, reconnecting") + } + return false + } else if err == syscall.EPIPE { + Debug("Detected broken pipe.", err) + return false + } + + return true +} + +func (c *TCPClient) Send(data []byte) (response []byte, err error) { + // Don't exit on panic + defer func() { + if r := recover(); r != nil { + Debug("[TCPClient]", r, string(data)) + + if _, ok := r.(error); !ok { + log.Println("[TCPClient] Failed to send request: ", string(data)) + log.Println("PANIC: pkg:", r, debug.Stack()) + } + } + }() + + if c.conn == nil || !c.isAlive() { + Debug("[TCPClient] Connecting:", c.baseURL) + if err = c.Connect(); err != nil { + log.Println("[TCPClient] Connection error:", err) + return + } + } + + timeout := time.Now().Add(c.config.Timeout) + + c.conn.SetWriteDeadline(timeout) + + if c.config.Debug { + Debug("[TCPClient] Sending:", string(data)) + } + + if _, err = c.conn.Write(data); err != nil { + Debug("[TCPClient] Write error:", err, c.baseURL) + return + } + + var readBytes, n int + var currentChunk []byte + timeout = time.Now().Add(c.config.Timeout) + + for { + c.conn.SetReadDeadline(timeout) + + if readBytes < len(c.respBuf) { + n, err = c.conn.Read(c.respBuf[readBytes:]) + readBytes += n + + if err != nil { + if err == io.EOF { + err = nil + } + break + } + } else { + if currentChunk == nil { + currentChunk = make([]byte, readChunkSize) + } + + n, err = c.conn.Read(currentChunk) + + if err == io.EOF { + break + } else if err != nil { + Debug("[TCPClient] Read the whole body error:", err, c.baseURL) + break + } + + readBytes += int(n) + } + + if readBytes >= maxResponseSize { + Debug("[TCPClient] Body is more than the max size", maxResponseSize, + c.baseURL) + break + } + + // For following chunks expect less timeout + timeout = time.Now().Add(c.config.Timeout / 5) + } + + if err != nil { + Debug("[TCPClient] Response read error", err, c.conn, readBytes) + return + } + + if readBytes > len(c.respBuf) { + readBytes = len(c.respBuf) + } + + payload := make([]byte, readBytes) + copy(payload, c.respBuf[:readBytes]) + + if c.config.Debug { + Debug("[TCPClient] Received:", string(payload)) + } + + return payload, err +} From 1d811f34db18723fa73a0b575fe761cee76bec82 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 13 Jul 2016 20:29:36 +0300 Subject: [PATCH 33/79] Update LICENSE.txt --- LICENSE.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/LICENSE.txt b/LICENSE.txt index 4d69dfd..cdc94a5 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2,6 +2,18 @@ Gor is an Open Source project licensed under the terms of the LGPLv3 license. Please see for license text. +As a special exception to the GNU Lesser General Public License version 3 +("LGPL3"), the copyright holders of this Library give you permission to +convey to a third party a Combined Work that links statically or dynamically +to this Library without providing any Minimal Corresponding Source or +Minimal Application Code as set out in 4d or providing the installation +information set out in section 4e, provided that you comply with the other +provisions of LGPL3 and provided that you meet, for the Application the +terms and conditions of the license(s) which apply to the Application. + +TLDR: You are free to use Gor subpackages like `byteutils` or `proto` in your commercial projects. + + Gor Pro has a commercial-friendly license allowing private forks and modifications of Gor. Please see http://gortool.com/#pro for more detail. You can find the commercial license terms in COMM-LICENSE. From 2e768f776d2da5f1d5c655635fa8eaa709416f7d Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 13 Jul 2016 20:45:44 +0300 Subject: [PATCH 34/79] Fix versioning --- Makefile | 12 ++++++------ s3/index.html | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 s3/index.html diff --git a/Makefile b/Makefile index f7807d8..b936219 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,25 @@ SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_null.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go tcp_client.go output_binary.go -SOURCE_PATH = /go/src/github.com/buger/gor/ +SOURCE_PATH = /go/src/github.com/buger/gor-pro/ PORT = 8000 FADDR = :8000 RUN = docker run -v `pwd`:$(SOURCE_PATH) -p 0.0.0.0:$(PORT):$(PORT) -t -i gor BENCHMARK = BenchmarkRAWInput TEST = TestRawListenerBench VERSION = DEV-$(shell date +%s) -LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\"" -MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)" +LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)_PRO -extldflags \"-static\"" +MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)_PRO" FADDR = ":8000" release: release-x64 release-mac release-x64: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build -o gor $(LDFLAGS) && tar -czf gor_$(VERSION)_PRO_x64.tar.gz gor && rm gor release-x86: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build -o gor $(LDFLAGS) && tar -czf gor_$(VERSION)_PRO_x86.tar.gz gor && rm gor release-mac: - go build $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_mac.tar.gz gor && rm gor + go build -o gor $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_PRO_mac.tar.gz gor && rm gor build: docker build -t gor . diff --git a/s3/index.html b/s3/index.html new file mode 100644 index 0000000..5557576 --- /dev/null +++ b/s3/index.html @@ -0,0 +1,47 @@ + + + + Gor PRO + + + + + +

Gor PRO releases

+

See releases page on GitHub for changelog

+ +

v0.14.1

+ + + \ No newline at end of file From a4826fcff914a8d9874b09daa42e4e0d1a072ec2 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 30 Jul 2016 16:42:40 +0300 Subject: [PATCH 35/79] Add support for tcp sessions --- emitter.go | 21 ++++-- emitter_test.go | 69 ++++++++++++++++++- output_http.go | 107 ++++++++++++++++++++++++----- output_http_test.go | 53 +++++++++++++- protocol.go | 24 +++++-- settings.go | 3 + test_input.go | 20 ++++-- vendor/github.com/bmizerany/assert | 1 + 8 files changed, 266 insertions(+), 32 deletions(-) create mode 160000 vendor/github.com/bmizerany/assert diff --git a/emitter.go b/emitter.go index 9575484..fd79ef3 100644 --- a/emitter.go +++ b/emitter.go @@ -4,6 +4,7 @@ import ( "bytes" "io" "time" + "hash/fnv" ) // Start initialize loop for sending data from inputs to outputs @@ -87,13 +88,23 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { } if Settings.splitOutput { - // Simple round robin - writers[wIndex].Write(payload) + if Settings.recognizeTCPSessions { + hasher := fnv.New32a() + // First 20 bytes contain tcp session + id := payloadID(payload) + hasher.Write(id[:20]) - wIndex++ + wIndex = int(hasher.Sum32()) % len(writers) + writers[wIndex].Write(payload) + } else { + // Simple round robin + writers[wIndex].Write(payload) - if wIndex >= len(writers) { - wIndex = 0 + wIndex++ + + if wIndex >= len(writers) { + wIndex = 0 + } } } else { for _, dst := range writers { diff --git a/emitter_test.go b/emitter_test.go index aec1e30..6991d71 100644 --- a/emitter_test.go +++ b/emitter_test.go @@ -5,6 +5,7 @@ import ( "sync" "sync/atomic" "testing" + "bytes" ) func TestEmitter(t *testing.T) { @@ -31,7 +32,7 @@ func TestEmitter(t *testing.T) { close(quit) } -func TestEmitterRoundRobin(t *testing.T) { +func TestEmitterSplitRoundRobin(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) @@ -72,6 +73,72 @@ func TestEmitterRoundRobin(t *testing.T) { Settings.splitOutput = false } +func TestEmitterSplitSession(t *testing.T) { + wg1 := new(sync.WaitGroup) + wg2 := new(sync.WaitGroup) + wg1.Add(1000) + wg2.Add(1000) + + // Base uuids, only 1 letter changed + uuid1 := []byte("1234567890123456789a0000") + uuid2 := []byte("1234567890123456789d0000") + + quit := make(chan int) + + input := NewTestInput() + input.disableHeaders = true + + var counter1, counter2 int32 + + output1 := NewTestOutput(func(data []byte) { + atomic.AddInt32(&counter1, 1) + if !bytes.Equal(uuid1[:20], payloadID(data)[:20]) { + t.Errorf("All tcp sessions should have same id") + } + wg1.Done() + }) + + output2 := NewTestOutput(func(data []byte) { + atomic.AddInt32(&counter2, 1) + if !bytes.Equal(uuid2[:20], payloadID(data)[:20]) { + t.Errorf("All tcp sessions should have same id") + } + wg2.Done() + }) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output1, output2} + + Settings.splitOutput = true + Settings.recognizeTCPSessions = true + + go Start(quit) + + for i := 0; i < 1000; i++ { + // Keep session but randomize ACK + copy(uuid1[20:], randByte(4)) + input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n")) + } + + for i := 0; i < 1000; i++ { + // Keep session but randomize ACK + copy(uuid2[20:], randByte(4)) + input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n")) + } + + wg1.Wait() + wg2.Wait() + + close(quit) + + if counter1 != 1000 || counter2 != 1000 { + t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2) + } + + Settings.splitOutput = false + Settings.recognizeTCPSessions = false +} + func BenchmarkEmitter(b *testing.B) { wg := new(sync.WaitGroup) quit := make(chan int) diff --git a/output_http.go b/output_http.go index d442b83..73f516e 100644 --- a/output_http.go +++ b/output_http.go @@ -4,12 +4,54 @@ import ( "io" "sync/atomic" "time" + "fmt" "github.com/buger/gor/proto" ) +var _ = fmt.Println + const initialDynamicWorkers = 10 +type httpWorker struct { + output *HTTPOutput + client *HTTPClient + lastActivity time.Time + queue chan []byte + stop chan bool +} + +func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker { + client := NewHTTPClient(output.address, &HTTPClientConfig{ + FollowRedirects: output.config.redirectLimit, + Debug: output.config.Debug, + OriginalHost: output.config.OriginalHost, + Timeout: output.config.Timeout, + ResponseBufferSize: output.config.BufferSize, + }) + + w := &httpWorker{client: client} + if queue == nil { + w.queue = make(chan []byte, 100) + } else { + w.queue = queue + } + w.stop = make(chan bool) + + go func(){ + for { + select { + case payload := <-w.queue: + output.sendRequest(client, payload) + case <- w.stop: + return + } + } + }() + + return w +} + type response struct { payload []byte uuid []byte @@ -44,6 +86,8 @@ type HTTPOutput struct { // aligned at 64bit. See https://github.com/golang/go/issues/599 activeWorkers int64 + workerSessions map[string]*httpWorker + address string limit int queue chan []byte @@ -91,7 +135,12 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o.config.TrackResponses = true } - go o.workerMaster() + if Settings.recognizeTCPSessions { + o.workerSessions = make(map[string]*httpWorker, 100) + go o.sessionWorkerMaster() + } else { + go o.workerMaster() + } return o } @@ -110,6 +159,39 @@ func (o *HTTPOutput) workerMaster() { } } +func (o *HTTPOutput) sessionWorkerMaster() { + gc := time.Tick(time.Second) + + for { + select { + case p := <-o.queue: + id := payloadID(p) + sessionID := string(id[0:20]) + worker, ok := o.workerSessions[sessionID] + + if !ok { + atomic.AddInt64(&o.activeWorkers, 1) + + worker = newHTTPWorker(o, nil) + o.workerSessions[sessionID] = worker + } + + worker.queue <- p + worker.lastActivity = time.Now() + case <-gc: + now := time.Now() + + for id, w := range o.workerSessions { + if !w.lastActivity.IsZero() && now.Sub(w.lastActivity) >= 60 * time.Second { + w.stop <- true + delete(o.workerSessions, id) + atomic.AddInt64(&o.activeWorkers, -1) + } + } + } + } +} + func (o *HTTPOutput) startWorker() { client := NewHTTPClient(o.address, &HTTPClientConfig{ FollowRedirects: o.config.redirectLimit, @@ -119,31 +201,24 @@ func (o *HTTPOutput) startWorker() { ResponseBufferSize: o.config.BufferSize, }) - deathCount := 0 - atomic.AddInt64(&o.activeWorkers, 1) for { select { case data := <-o.queue: o.sendRequest(client, data) - deathCount = 0 - case <-time.After(time.Millisecond * 100): + case <-time.After(2 * time.Second): // When dynamic scaling enabled workers die after 2s of inactivity - if o.config.workers == 0 { - deathCount++ - } else { + if o.config.workers > 0 { continue } - if deathCount > 20 { - workersCount := atomic.LoadInt64(&o.activeWorkers) + workersCount := atomic.LoadInt64(&o.activeWorkers) - // At least 1 startWorker should be alive - if workersCount != 1 { - atomic.AddInt64(&o.activeWorkers, -1) - return - } + // At least 1 startWorker should be alive + if workersCount != 1 { + atomic.AddInt64(&o.activeWorkers, -1) + return } } } @@ -163,7 +238,7 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { o.queueStats.Write(len(o.queue)) } - if o.config.workers == 0 { + if !Settings.recognizeTCPSessions && o.config.workers == 0 { workersCount := atomic.LoadInt64(&o.activeWorkers) if len(o.queue) > int(workersCount) { diff --git a/output_http_test.go b/output_http_test.go index fd4d2c6..a3324a2 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -59,6 +59,10 @@ func TestHTTPOutput(t *testing.T) { wg.Wait() + if output.(*HTTPOutput).activeWorkers != 200 { + t.Error("Should create workers for each request", output.(*HTTPOutput).activeWorkers) + } + close(quit) Settings.modifierConfig = HTTPModifierConfig{} @@ -99,7 +103,7 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) { Settings.modifierConfig = HTTPModifierConfig{} } -func TestOutputHTTPSSL(t *testing.T) { +func TestHTTPOutputSSL(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) @@ -125,6 +129,53 @@ func TestOutputHTTPSSL(t *testing.T) { close(quit) } +func TestHTTPOutputSessions(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + input := NewTestInput() + input.disableHeaders = true + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + wg.Done() + })) + defer server.Close() + + Settings.recognizeTCPSessions = true + output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true}) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + + go Start(quit) + + uuid1 := []byte("1234567890123456789a0000") + uuid2 := []byte("1234567890123456789d0000") + + + for i := 0; i < 100; i++ { + wg.Add(1) // OPTIONS should be ignored + copy(uuid1[20:], randByte(4)) + input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n")) + } + + for i := 0; i < 100; i++ { + wg.Add(1) // OPTIONS should be ignored + copy(uuid2[20:], randByte(4)) + input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n")) + } + + wg.Wait() + + if output.(*HTTPOutput).activeWorkers != 2 { + t.Error("Should have only 2 workers", output.(*HTTPOutput).activeWorkers) + } + + close(quit) + + Settings.recognizeTCPSessions = false +} + func BenchmarkHTTPOutput(b *testing.B) { wg := new(sync.WaitGroup) quit := make(chan int) diff --git a/protocol.go b/protocol.go index 50d6943..2ac0f46 100644 --- a/protocol.go +++ b/protocol.go @@ -13,14 +13,18 @@ const ( ReplayedResponsePayload = '3' ) -func uuid() []byte { - b := make([]byte, 20) +func randByte(len int) []byte { + b := make([]byte, len / 2) rand.Read(b) - uuid := make([]byte, 40) - hex.Encode(uuid, b) + h := make([]byte, len) + hex.Encode(h, b) - return uuid + return h +} + +func uuid() []byte { + return randByte(24) } var payloadSeparator = "\n🐵🙈🙉\n" @@ -88,6 +92,16 @@ func payloadMeta(payload []byte) [][]byte { return bytes.Split(payload[:headerSize], []byte{' '}) } +func payloadID(payload []byte) []byte { + idx := bytes.IndexByte(payload[2:], ' ') + + if idx == -1 { + return []byte{} + } + + return payload[2: 2 + idx] +} + func isOriginPayload(payload []byte) bool { switch payload[0] { case RequestPayload, ResponsePayload: diff --git a/settings.go b/settings.go index dd0675c..24b6699 100644 --- a/settings.go +++ b/settings.go @@ -30,6 +30,7 @@ type AppSettings struct { stats bool splitOutput bool + recognizeTCPSessions bool inputDummy MultiOption outputDummy MultiOption @@ -77,6 +78,8 @@ func init() { flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.") + flag.BoolVar(&Settings.recognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.") + flag.Var(&Settings.inputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s") flag.Var(&Settings.outputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead") diff --git a/test_input.go b/test_input.go index 0f990e7..b903359 100644 --- a/test_input.go +++ b/test_input.go @@ -9,6 +9,7 @@ import ( // TestInput used for testing purpose, it allows emitting requests on demand type TestInput struct { data chan []byte + disableHeaders bool } // NewTestInput constructor for TestInput @@ -22,11 +23,21 @@ func NewTestInput() (i *TestInput) { func (i *TestInput) Read(data []byte) (int, error) { buf := <-i.data - header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1) - copy(data[0:len(header)], header) - copy(data[len(header):], buf) + if !i.disableHeaders { + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1) + copy(data[0:len(header)], header) + copy(data[len(header):], buf) - return len(buf) + len(header), nil + return len(buf) + len(header), nil + } else { + copy(data, buf) + return len(buf), nil + } +} + +// EmitGET emits GET request without headers +func (i *TestInput) EmitBytes(b []byte) { + i.data <- b } // EmitGET emits GET request without headers @@ -34,6 +45,7 @@ func (i *TestInput) EmitGET() { i.data <- []byte("GET / HTTP/1.1\r\n\r\n") } + // EmitPOST emits POST request with Content-Length func (i *TestInput) EmitPOST() { i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") diff --git a/vendor/github.com/bmizerany/assert b/vendor/github.com/bmizerany/assert new file mode 160000 index 0000000..b7ed37b --- /dev/null +++ b/vendor/github.com/bmizerany/assert @@ -0,0 +1 @@ +Subproject commit b7ed37b82869576c289d7d97fb2bbd8b64a0cb28 From d34c27c3deeaf027826dbfa8de9a13fef51d54c8 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 31 Jul 2016 20:16:01 +0300 Subject: [PATCH 36/79] Track FIN packets to check for closed connection (#350) * Track FIN packets to check for closed connection * Add tests for FIN / fix bugs * Fmt changes --- http_client_test.go | 31 ++++++++++++++ input_raw_test.go | 49 ++++++++++++++++++++++ raw_socket_listener/listener.go | 24 ++++++----- raw_socket_listener/listener_test.go | 40 ++++++++++++++++++ raw_socket_listener/tcp_message.go | 55 +++++++++++++++++++------ raw_socket_listener/tcp_message_test.go | 7 ++-- raw_socket_listener/tcp_packet.go | 8 ++++ 7 files changed, 186 insertions(+), 28 deletions(-) diff --git a/http_client_test.go b/http_client_test.go index d93984a..79c71e7 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -87,6 +87,37 @@ func TestHTTPClientSend(t *testing.T) { wg.Wait() } +func TestHTTPClientResonseByClose(t *testing.T) { + wg := new(sync.WaitGroup) + + payload := []byte("GET / HTTP/1.1\r\n\r\n") + ln, _ := net.Listen("tcp", ":0") + go func(){ + for { + conn, _ := ln.Accept() + buf := make([]byte, 4096) + conn.Read(buf) + + conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n")) + conn.Write([]byte("ab")) + conn.Close() + + wg.Done() + } + }() + + client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{Debug: true}) + + wg.Add(1) + resp, _ := client.Send(payload) + + if !bytes.Equal(resp, []byte("HTTP/1.1 200 OK\r\n\r\nab")) { + t.Error("Should return valid response", string(resp)) + } + + wg.Wait() +} + // https://github.com/buger/gor/issues/184 func TestHTTPClientResponseBuffer(t *testing.T) { testCases := []struct { diff --git a/input_raw_test.go b/input_raw_test.go index b8fe1b1..a40b0fa 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -79,6 +79,55 @@ func TestRAWInputIPv4(t *testing.T) { } wg.Wait() + + close(quit) +} + +func TestRAWInputNoKeepAlive(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + listener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + origin := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("a")) + w.Write([]byte("b")) + }), + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + origin.SetKeepAlivesEnabled(false) + go origin.Serve(listener) + defer listener.Close() + + originAddr := listener.Addr().String() + + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "") + defer input.Close() + + output := NewTestOutput(func(data []byte) { + wg.Done() + }) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + + client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{}) + + go Start(quit) + + for i := 0; i < 100; i++ { + // request + response + wg.Add(2) + client.Get("/") + time.Sleep(2 * time.Millisecond) + } + + wg.Wait() + close(quit) } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index a4c88fd..f85597b 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -181,6 +181,7 @@ func (t *Listener) dispatchMessage(message *TCPMessage) { delete(t.respAliases, message.Ack) delete(t.respWithoutReq, message.Ack) } + return } @@ -337,7 +338,7 @@ func (t *Listener) readPcap() { var allAddr []string for _, dc := range devices { for _, addr := range dc.Addresses { - allAddr = append(allAddr, "(dst host " + addr.IP.String() + " and src host " + addr.IP.String() + ")") + allAddr = append(allAddr, "(dst host "+addr.IP.String()+" and src host "+addr.IP.String()+")") } } @@ -441,7 +442,7 @@ func (t *Listener) readPcap() { } // Invalid length - if int(ihl * 4) > ipLength { + if int(ihl*4) > ipLength { continue } @@ -452,7 +453,7 @@ func (t *Listener) readPcap() { continue } - data = data[ihl * 4:] + data = data[ihl*4:] } else { // Truncated IP info if len(data) < 40 { @@ -471,10 +472,11 @@ func (t *Listener) readPcap() { } dataOffset := (data[12] & 0xF0) >> 4 + isFIN := data[13]&0x01 != 0 // We need only packets with data inside // Check that the buffer is larger than the size of the TCP header - if len(data) > int(dataOffset*4) { + if len(data) > int(dataOffset*4) || isFIN { if !bpfSupported { destPort := binary.BigEndian.Uint16(data[2:4]) srcPort := binary.BigEndian.Uint16(data[0:2]) @@ -555,16 +557,16 @@ func (t *Listener) readPcapFile() { var addr, data []byte if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil { - tcp, _ := tcpLayer.(*layers.TCP) - data = append(tcp.LayerContents(), tcp.LayerPayload()...) - copy(data[2:4], []byte{0, 1}) + tcp, _ := tcpLayer.(*layers.TCP) + data = append(tcp.LayerContents(), tcp.LayerPayload()...) + copy(data[2:4], []byte{0, 1}) } else { continue } if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil { - ip, _ := ipLayer.(*layers.IPv4) - addr = ip.SrcIP + ip, _ := ipLayer.(*layers.IPv4) + addr = ip.SrcIP } else if ipLayer = packet.Layer(layers.LayerTypeIPv6); ipLayer != nil { ip, _ := ipLayer.(*layers.IPv6) addr = ip.SrcIP @@ -763,13 +765,13 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { // If message contains only single packet immediately dispatch it if message.complete { if isIncoming { - // log.Println("I'm finished", string(message.Bytes()), message.ResponseID, t.messages) if t.trackResponse { if resp, ok := t.messages[message.ResponseID]; ok { - t.dispatchMessage(message) if resp.complete { t.dispatchMessage(resp) } + + t.dispatchMessage(message) } } else { t.dispatchMessage(message) diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index b66b23a..72e01de 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -46,6 +46,46 @@ func TestRawListenerInput(t *testing.T) { } } +func TestRawListenerInputResponseByClose(t *testing.T) { + var req, resp *TCPMessage + + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) + defer listener.Close() + + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) + + respAck := reqPacket.Seq + uint32(len(reqPacket.Data)) + respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd")) + finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte("")) + finPacket.IsFIN = true + + listener.packetsChan <- reqPacket.Dump() + listener.packetsChan <- respPacket.Dump() + listener.packetsChan <- finPacket.Dump() + + select { + case req = <-listener.messagesChan: + case <-time.After(time.Millisecond): + t.Error("Should return request immediately") + return + } + + if !req.IsIncoming { + t.Error("Should be request") + } + + select { + case resp = <-listener.messagesChan: + case <-time.After(20 * time.Millisecond): + t.Error("Should return response immediately") + return + } + + if resp.IsIncoming { + t.Error("Should be response") + } +} + func TestRawListenerInputWithoutResponse(t *testing.T) { var req *TCPMessage diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 15837bd..a394153 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -72,7 +72,7 @@ func (t *TCPMessage) BodySize() (size int) { size += len(proto.Body(t.packets[t.headerPacket].Data)) - for _, p := range t.packets[t.headerPacket + 1:] { + for _, p := range t.packets[t.headerPacket+1:] { size += len(p.Data) } @@ -145,7 +145,17 @@ func (t *TCPMessage) checkSeqIntegrity() { t.seqMissing = false } - for i, p := range t.packets { + offset := len(t.packets) - 1 + + if t.packets[offset].IsFIN { + offset-- + } + + for i, p := range t.packets[:offset] { + if p.IsFIN { + continue + } + // If final packet if len(t.packets) == i+1 { t.seqMissing = false @@ -228,6 +238,15 @@ func (t *TCPMessage) checkIfComplete() { if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 { t.complete = true } + default: + if len(t.packets) == 0 { + return + } + + last := t.packets[len(t.packets)-1] + if last.IsFIN { + t.complete = true + } } } } @@ -302,10 +321,11 @@ func (t *TCPMessage) updateMethodType() { type httpBodyType uint8 const ( - httpBodyNotSet httpBodyType = 0 - httpBodyEmpty httpBodyType = 1 - httpBodyContentLength httpBodyType = 2 - httpBodyChunked httpBodyType = 3 + httpBodyNotSet httpBodyType = 0 + httpBodyEmpty httpBodyType = 1 + httpBodyContentLength httpBodyType = 2 + httpBodyChunked httpBodyType = 3 + httpBodyConnectionClose httpBodyType = 4 ) func (t *TCPMessage) updateBodyType() { @@ -326,7 +346,7 @@ func (t *TCPMessage) updateBodyType() { t.bodyType = httpBodyEmpty return case httpMethodWithBody: - var lengthB, encB []byte + var lengthB, encB, connB []byte for _, p := range t.packets[:t.headerPacket+1] { lengthB = proto.Header(p.Data, []byte("Content-Length")) @@ -340,12 +360,21 @@ func (t *TCPMessage) updateBodyType() { t.bodyType = httpBodyContentLength t.contentLength, _ = strconv.Atoi(string(lengthB)) return - } else { - for _, p := range t.packets[:t.headerPacket+1] { - encB = proto.Header(p.Data, []byte("Transfer-Encoding")) + } - if len(encB) > 0 { - t.bodyType = httpBodyChunked + for _, p := range t.packets[:t.headerPacket+1] { + encB = proto.Header(p.Data, []byte("Transfer-Encoding")) + + if len(encB) > 0 { + t.bodyType = httpBodyChunked + return + } + + for _, p := range t.packets[:t.headerPacket+1] { + connB = proto.Header(p.Data, []byte("Connection")) + + if len(connB) > 0 && bytes.Equal(connB, []byte("close")) { + t.bodyType = httpBodyConnectionClose return } } @@ -363,7 +392,7 @@ const ( httpExpect100Continue httpExpectType = 2 ) -var bExpectHeader = []byte("Expect:") +var bExpectHeader = []byte("Expect") var bExpect100Value = []byte("100-continue") func (t *TCPMessage) check100Continue() { diff --git a/raw_socket_listener/tcp_message_test.go b/raw_socket_listener/tcp_message_test.go index 0af3bd9..781a824 100644 --- a/raw_socket_listener/tcp_message_test.go +++ b/raw_socket_listener/tcp_message_test.go @@ -216,12 +216,11 @@ func TestTCPMessageBodyType(t *testing.T) { } } - func TestTCPMessageBodySize(t *testing.T) { testCases := []struct { - direction bool - payloads []string - expectedSize int + direction bool + payloads []string + expectedSize int }{ {true, []string{"GET / HTTP/1.1\r\n\r\n"}, 0}, {true, []string{"POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab"}, 2}, diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index d53834e..0e3e832 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -33,6 +33,7 @@ type TCPPacket struct { Ack uint32 OrigAck uint32 DataOffset uint8 + IsFIN bool Raw []byte Data []byte @@ -71,6 +72,7 @@ func (t *TCPPacket) ParseBasic() { t.Seq = binary.BigEndian.Uint32(t.Raw[4:8]) t.Ack = binary.BigEndian.Uint32(t.Raw[8:12]) t.DataOffset = (t.Raw[12] & 0xF0) >> 4 + t.IsFIN = t.Raw[13]&0x01 != 0 // log.Println("DataOffset:", t.DataOffset, t.DestPort, t.SrcPort, t.Seq, t.Ack) @@ -90,6 +92,11 @@ func (t *TCPPacket) Dump() []byte { binary.BigEndian.PutUint32(tcpBuf[8:12], t.Ack) tcpBuf[12] = 64 + + if t.IsFIN { + tcpBuf[13] = tcpBuf[13] | 0x01 + } + copy(tcpBuf[16:], t.Data) return buf @@ -109,6 +116,7 @@ func (t *TCPPacket) String() string { "Sequence:" + strconv.Itoa(int(t.Seq)), "Acknowledgment:" + strconv.Itoa(int(t.Ack)), "Header len:" + strconv.Itoa(int(t.DataOffset)), + "FIN:" + strconv.FormatBool(t.IsFIN), "Data size:" + strconv.Itoa(len(t.Data)), "Data:" + string(t.Data[:maxLen]), From f0acd31d88dd78e8d32421e07f444535cf583ada Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 31 Jul 2016 15:32:52 +0300 Subject: [PATCH 37/79] Disable --input-http flag until I decide how to do it properly --- settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.go b/settings.go index 1b6104e..13427e2 100644 --- a/settings.go +++ b/settings.go @@ -112,7 +112,7 @@ func init() { flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command") - flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") + // flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") From 07fa6d9a7c564e356aeeaee081585ee735fdea00 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 1 Aug 2016 18:05:52 +0300 Subject: [PATCH 38/79] Force Go DNS resolver --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c0c1e45..a35dcb9 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,10 @@ FADDR = ":8000" release: release-x64 release-mac release-x64: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor release-x86: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor release-mac: go build $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_mac.tar.gz gor && rm gor From f1f8e2f2a48803ef5a4139ecb798fb2b600821b0 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 1 Aug 2016 19:17:05 +0300 Subject: [PATCH 39/79] Fix http-disallow-header option --- settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.go b/settings.go index 13427e2..04828e7 100644 --- a/settings.go +++ b/settings.go @@ -145,7 +145,7 @@ func init() { flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") - flag.Var(&Settings.modifierConfig.headerFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"") + flag.Var(&Settings.modifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"") flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%") flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead") From 25b998fa3f1ae9100ade6329c44c6a030522159a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 1 Aug 2016 19:23:14 +0300 Subject: [PATCH 40/79] Fix run command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a35dcb9..fb1e399 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ profile_test: # Used mainly for debugging, because docker container do not have access to parent machine ports run: - $(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --input-http 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor + $(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor run-2: sudo -E go run $(SOURCE) --input-dummy="" --output-tcp localhost:27001 --verbose --debug From e38f6a580186ac0474692304aa924227dbb765bc Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 2 Aug 2016 12:55:02 +0300 Subject: [PATCH 41/79] Do not quite on flush error --- output_file.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/output_file.go b/output_file.go index 044068d..9156ba2 100644 --- a/output_file.go +++ b/output_file.go @@ -8,6 +8,7 @@ import ( "log" "os" "path/filepath" + "runtime/debug" "sort" "strconv" "strings" @@ -208,6 +209,13 @@ func (o *FileOutput) Write(data []byte) (n int, err error) { } func (o *FileOutput) flush() { + // Don't exit on panic + defer func() { + if r := recover(); r != nil { + log.Println("PANIC while file flush: ", r, o, string(debug.Stack())) + } + }() + defer o.mu.Unlock() o.mu.Lock() From fbebe72f02b669a86fb1e7984b1cd0688f82bf39 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 3 Aug 2016 16:56:50 +0300 Subject: [PATCH 42/79] Fix multi-packet splited headers --- proto/proto.go | 121 +++++++++++++++++++++++++++++ proto/proto_test.go | 40 ++++++++++ raw_socket_listener/tcp_message.go | 83 +++++++++++++------- 3 files changed, 216 insertions(+), 28 deletions(-) diff --git a/proto/proto.go b/proto/proto.go index 186926f..0bdb819 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -160,6 +160,127 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, return } +// Works only with ASCII +func HeadersEqual(h1 []byte, h2 []byte) bool { + if len(h1) != len(h2) { + return false + } + + for i, c1 := range h1 { + c2 := h2[i] + + switch int(c1) - int(c2) { + case 0, 32, -32: + default: + return false + } + } + + return true +} + +// Parsing headers from multiple payloads +func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) { + + hS := [2]int{0, 0} + hE := [2]int{-1, -1} + vS := [2]int{-1, -1} + vE := [2]int{-1, -1} + + i := 0 + pIdx := 0 + lineBreaks := 0 + + for { + if len(payloads)-1 < pIdx { + break + } + + p := payloads[pIdx] + + if len(p)-1 < i { + pIdx++ + i = 0 + continue + } + + switch p[i] { + case '\r', '\n': + lineBreaks++ + + // End of headers + if lineBreaks == 4 { + return + } + + if lineBreaks > 1 { + break + } + + vE = [2]int{pIdx, i} + + if vS[1] != -1 && vE[1] != -1 && + hS[1] != -1 && hE[1] != -1 { + + var header, value []byte + + phS, phE, pvS, pvE := payloads[hS[0]], payloads[hE[0]], payloads[vS[0]], payloads[vE[0]] + + // If in same payload + if hS[0] == hE[0] { + header = phS[hS[1]:hE[1]] + } else { + header = make([]byte, len(phS)-hS[1]+hE[1]) + copy(header, phS[hS[1]:]) + copy(header[len(phS)-hS[1]:], phE[:hE[1]]) + } + + if vS[0] == vE[0] { + value = pvS[vS[1]:vE[1]] + } else { + value = make([]byte, len(pvS)-vS[1]+vE[1]) + copy(value, pvS[vS[1]:]) + copy(value[len(pvS)-vS[1]:], pvE[:vE[1]]) + } + + if !cb(header, value) { + return + } + } + + // Header found, reset values + vS = [2]int{-1, -1} + vE = [2]int{-1, -1} + hS = [2]int{-1, -1} + hE = [2]int{-1, -1} + case ':': + hE = [2]int{pIdx, i} + default: + lineBreaks = 0 + + if hS[1] == -1 { + hS = [2]int{pIdx, i} + } else { + if hE[1] == -1 { + break + } + + if vS[1] == -1 { + if p[i] == ' ' { + break + } + + vS = [2]int{pIdx, i} + } + } + } + + i++ + } + + return +} + // Header returns header value, if header not found, value will be blank func Header(payload, name []byte) []byte { val, _, _, _, _ := header(payload, name) diff --git a/proto/proto_test.go b/proto/proto_test.go index 897bc4f..32da8c9 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -2,6 +2,7 @@ package proto import ( "bytes" + "reflect" "testing" ) @@ -123,6 +124,45 @@ func TestDeleteHeader(t *testing.T) { } } +func TestParseHeaders(t *testing.T) { + payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Chrome\r\n\r\n"), []byte("Fake-Header: asda")} + + headers := make(map[string]string) + + ParseHeaders(payload, func(header []byte, value []byte) bool { + headers[string(header)] = string(value) + return true + }) + + expected := map[string]string{ + "Content-Length": "7", + "Host": "www.w3.org", + "User-Agent": "Chrome", + } + if !reflect.DeepEqual(headers, expected) { + t.Error("Headers do not properly parsed", headers) + } +} + +func TestHeaderEquals(t *testing.T) { + tests := []struct { + h1 string + h2 string + equals bool + }{ + {"Content-Length", "content-length", true}, + {"content-length", "Content-Length", true}, + {"content-Pength", "Content-Length", false}, + {"Host", "Content-Length", false}, + } + + for _, tc := range tests { + if HeadersEqual([]byte(tc.h1), []byte(tc.h2)) != tc.equals { + t.Error(tc) + } + } +} + func TestPath(t *testing.T) { var path, payload []byte diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index a394153..7a5185c 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -55,6 +55,15 @@ func NewTCPMessage(Seq, Ack uint32, IsIncoming bool) (msg *TCPMessage) { return } +func (t *TCPMessage) packetsData() (d [][]byte) { + d = make([][]byte, len(t.packets)) + for i, p := range t.packets { + d[i] = p.Data + } + + return +} + // Bytes return message content func (t *TCPMessage) Bytes() (output []byte) { for _, p := range t.packets { @@ -149,6 +158,10 @@ func (t *TCPMessage) checkSeqIntegrity() { if t.packets[offset].IsFIN { offset-- + + if offset < 0 { + return + } } for i, p := range t.packets[:offset] { @@ -348,36 +361,44 @@ func (t *TCPMessage) updateBodyType() { case httpMethodWithBody: var lengthB, encB, connB []byte - for _, p := range t.packets[:t.headerPacket+1] { - lengthB = proto.Header(p.Data, []byte("Content-Length")) - - if len(lengthB) > 0 { - break + proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{ + if proto.HeadersEqual(header, []byte("Content-Length")) { + lengthB = value + return false } - } + + if proto.HeadersEqual(header, []byte("Transfer-Encoding")) { + encB = value + return false + } + + if proto.HeadersEqual(header, []byte("Connection")) { + connB = value + return false + } + + return true + }) if len(lengthB) > 0 { - t.bodyType = httpBodyContentLength t.contentLength, _ = strconv.Atoi(string(lengthB)) + + if t.contentLength == 0 { + t.bodyType = httpBodyEmpty + } else { + t.bodyType = httpBodyContentLength + } return } - for _, p := range t.packets[:t.headerPacket+1] { - encB = proto.Header(p.Data, []byte("Transfer-Encoding")) + if len(encB) > 0 { + t.bodyType = httpBodyChunked + return + } - if len(encB) > 0 { - t.bodyType = httpBodyChunked - return - } - - for _, p := range t.packets[:t.headerPacket+1] { - connB = proto.Header(p.Data, []byte("Connection")) - - if len(connB) > 0 && bytes.Equal(connB, []byte("close")) { - t.bodyType = httpBodyConnectionClose - return - } - } + if len(connB) > 0 && bytes.Equal(connB, []byte("close")) { + t.bodyType = httpBodyConnectionClose + return } } @@ -414,13 +435,19 @@ func (t *TCPMessage) check100Continue() { return } - for _, p := range t.packets[:t.headerPacket+1] { - if h := proto.Header(p.Data, bExpectHeader); len(h) > 0 { - if bytes.Equal(bExpect100Value, h) { - t.expectType = httpExpect100Continue - } - return + var expectB []byte + proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{ + if proto.HeadersEqual(header, bExpectHeader) { + expectB = value + return false } + + return true + }) + + if len(expectB) > 0 && bytes.Equal(bExpect100Value, expectB) { + t.expectType = httpExpect100Continue + return } t.expectType = httpExpectEmpty From 392d9a2f0ed0858766f1215395bde1c94c5d3efc Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 5 Aug 2016 14:09:20 +0300 Subject: [PATCH 43/79] Update LICENSE.txt --- LICENSE.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index cdc94a5..fff52dc 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Gor is an Open Source project licensed under the terms of +GoReplay is an Open Source project licensed under the terms of the LGPLv3 license. Please see for license text. @@ -14,6 +14,6 @@ terms and conditions of the license(s) which apply to the Application. TLDR: You are free to use Gor subpackages like `byteutils` or `proto` in your commercial projects. -Gor Pro has a commercial-friendly license allowing private forks -and modifications of Gor. Please see http://gortool.com/#pro for +GoReplay Pro has a commercial-friendly license allowing private forks +and modifications of GoReplay. Please see https://goreplay.org/pro.html for more detail. You can find the commercial license terms in COMM-LICENSE. From 25ded56a5902dcd207c1156769b7bdd6f82a7cbc Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 5 Aug 2016 14:14:37 +0300 Subject: [PATCH 44/79] Update COMM-LICENSE --- COMM-LICENSE | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/COMM-LICENSE b/COMM-LICENSE index 7d8d82b..b5d8678 100644 --- a/COMM-LICENSE +++ b/COMM-LICENSE @@ -2,7 +2,7 @@ END-USER LICENSE AGREEMENT ------------------------------------------------------------------------------ -IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND REPLAY SOFTWARE. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOR PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT. +IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND GoReplay LLC. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT. ------------------------------------------------------------------------------ @@ -12,21 +12,19 @@ In order to use the Software under this Agreement, you must receive a “Source 1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold. -1.2 Unlimited Organization License. If you purchased an Organization License (included with the Gor Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Gor server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers. +1.2 Unlimited Organization License. If you purchased an Organization License (included with the GoReplay Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may concurrently run the software on an unlimited number of Hosts. -1.3 Limited Organization License. If you purchased an Organization License (included with the Gor Enterprise Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a running Gor instance which intercept or replay traffic. In order to run additional Workers, you must purchase an additional allowance from Replay Software. +1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform GoReplay LLC of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe GoReplay LLC any royalties for your distribution of the Software in accordance with this EULA. -1.4 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Replay Software of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Replay Software any royalties for your distribution of the Software in accordance with this EULA. +1.4 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product. -1.5 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product. +1.5 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). -1.6 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). - -2. Modifications. Replay Software shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software. +2. Modifications. GoReplay LLC shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software. 3. Restricted Uses. -3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Replay Software; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Replay Software in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Replay Software; (j) use the Software to develop a product which is competitive with any Replay Software product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Replay Software in writing. If your unique Source URL is ever published, Replay Software reserves the right to terminate your access without notice. +3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by GoReplay LLC; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by GoReplay LLC in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by GoReplay LLC; (j) use the Software to develop a product which is competitive with any GoReplay LLC product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by GoReplay LLC in writing. If your unique Source URL is ever published, GoReplay LLC reserves the right to terminate your access without notice. 3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF. @@ -34,34 +32,34 @@ The Open Source version of the Software (“LGPL Version”) is licensed under the terms of the GNU Lesser General Public License versions 3.0 (“LGPL”) and not under this EULA. -4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Replay Software and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. +4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, GoReplay LLC and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. -5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Replay Software). +5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of GoReplay LLC). -6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the Gor Pro support page (https://github.com/buger/gor/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription. +6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/gor/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription. 7. Term of Agreement. -7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Replay Software may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise. +7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that GoReplay LLC may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise. 7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof. -7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from Replay Software. +7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from GoReplay LLC. -8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. Replay Software does not warrant that the Software will be free of bugs, errors, viruses or other defects, and Replay Software shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for Replay Software to do so. -To the maximum extent permitted by applicable law, Replay Software disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose. +8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. GoReplay LLC does not warrant that the Software will be free of bugs, errors, viruses or other defects, and GoReplay LLC shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for GoReplay LLC to do so. +To the maximum extent permitted by applicable law, GoReplay LLC disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose. 9. Limitation of Liability. -In no event will Replay Software be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if Replay Software has been advised of the possibility of such damages. +In no event will GoReplay LLC be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if GoReplay LLC has been advised of the possibility of such damages. -In no event will Replay Software' liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit. +In no event will GoReplay LLC liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit. -10. Remedies. Your exclusive remedy and Replay Software’ entire liability for breach of this Agreement shall be limited, at Replay Software’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to Replay Software, payable in accordance with Replay Software' refund policy. +10. Remedies. Your exclusive remedy and GoReplay LLC’ entire liability for breach of this Agreement shall be limited, at GoReplay LLC’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to GoReplay LLC, payable in accordance with GoReplay LLC' refund policy. 11. Acknowledgements. -11.1 Consent to the Use of Data. You agree that Replay Software and its affiliates may collect and use technical information gathered as part of the product support services. Replay Software may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you. +11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you. 11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software. @@ -74,13 +72,13 @@ In no event will Replay Software' liability exceed the Software license price as 13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral. -13.2 Amendment. Replay Software reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/gor/blob/master/COMM-LICENSE. +13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/gor/blob/master/COMM-LICENSE. -13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Replay Software and any attempted assignment without such consent shall be void. +13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of GoReplay LLC and any attempted assignment without such consent shall be void. 13.4 Export Compliance. You agree to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software. -13.5 Indemnification. You agree to defend, indemnify, and hold harmless Replay Software from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement. +13.5 Indemnification. You agree to defend, indemnify, and hold harmless GoReplay LLC from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement. 13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts. @@ -92,4 +90,4 @@ In no event will Replay Software' liability exceed the Software license price as 13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs. -14. Contact Information. If you have any questions about this EULA, or if you want to contact Replay Software for any reason, please direct correspondence to info@gortool.com +14. Contact Information. If you have any questions about this EULA, or if you want to contact GoReplay LLC for any reason, please direct correspondence to info@gortool.com From 544d9402f08541dd885dd24d29bc9410d07605fb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 8 Aug 2016 18:09:14 +0300 Subject: [PATCH 45/79] Disable forced flush --- output_file.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/output_file.go b/output_file.go index 9156ba2..a50b905 100644 --- a/output_file.go +++ b/output_file.go @@ -53,14 +53,6 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput { o.config = config o.updateName() - // Force flushing every minute - go func() { - for { - time.Sleep(o.config.flushInterval) - o.flush() - } - }() - go func() { for { time.Sleep(time.Second) From ecd7e3a5e508886afd15ed670d72aac5dde9e370 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 8 Aug 2016 19:35:11 +0300 Subject: [PATCH 46/79] Update README.md --- README.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1d0af52..ef70b72 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,17 @@ ![Go Replay](http://i.imgur.com/ZG2ki5n.png) +## https://goreplay.org/ + ## About -Gor is an open-source tool for capturing and replaying live HTTP traffic into a test environment in order to continuously test your system with real data. It can be used to increase confidence in code deployments, configuration changes and infrastructure changes. +GoReplay is the simplest and safest way to test your app using real traffic before you put it into production. + +As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. + +GoReplay increases your confidence in code deployments, configuration changes and infrastructure changes. Did we mention that no coding is required? + -Now you can test your code on real user sessions in an automated and repeatable fashion. -**No more falling down in production!** Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address. @@ -23,14 +28,14 @@ Download latest binary from https://github.com/buger/gor/releases or [compile by The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump. If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`. -See the our wiki and especially [Getting started](https://github.com/buger/gor/wiki/Getting-Started) wiki page for more info. +See the our [documentation](https://github.com/buger/gor/wiki/) and [Getting started](https://github.com/buger/gor/wiki/Getting-Started) page for more info. ## Newsletter Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project. ## Want to Upgrade? -I also sell Gor Pro, extensions to Gor which provide more features, a commercial-friendly license and allow you to support high quality open source development all at the same time. Please see the Gor [homepage](https://gortool.com/) for more detail. +We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP sessions replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. ## Problems? @@ -39,14 +44,6 @@ If you have a problem, please review the [FAQ](https://github.com/buger/gor/wiki All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com). If you have a private question feel free to send email to support@gortool.com. -Useful resources: - -* Product documentation is in the [wiki](http://github.com/buger/gor/wiki). -* Release announcements are made to the [@buger](http://twitter.com/buger) Twitter account and our [newsleter](https://tinyletter.com/gor) - - -If you need commercial support read more about Pro and Enterprise versions at our site [https://gortool.com/](https://gortool.com/) - ## Contributing @@ -69,7 +66,7 @@ If you need commercial support read more about Pro and Enterprise versions at ou * [Granify](http://granify.com) - AI backed SaaS solution that enables online retailers to maximise their sales * And many more! -If you are using Gor we are happy add you to the list and share your story, just write to: hello@gortool.com +If you are using Gor we are happy add you to the list and share your story, just write to: hello@goreplay.org ## Author From dac3b171a37a7896218cc1973bac953cc3f50dd4 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 10 Aug 2016 13:35:19 +0300 Subject: [PATCH 47/79] Fix test --- Makefile | 9 +++++---- input_raw_test.go | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index ca8c1f6..6809af0 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,8 @@ SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go SOURCE_PATH = /go/src/github.com/buger/gor-pro/ PORT = 8000 FADDR = :8000 -RUN = docker run -v `pwd`:$(SOURCE_PATH) -p 0.0.0.0:$(PORT):$(PORT) -t -i gor +CONTAINER=gor-pro +RUN = docker run -v `pwd`:$(SOURCE_PATH) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER) BENCHMARK = BenchmarkRAWInput TEST = TestRawListenerBench VERSION = DEV-$(shell date +%s) @@ -13,16 +14,16 @@ FADDR = ":8000" release: release-x64 release-mac release-x64: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor release-x86: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor release-mac: go build -o gor $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_PRO_mac.tar.gz gor && rm gor build: - docker build -t gor . + docker build -t $(CONTAINER) . profile: diff --git a/input_raw_test.go b/input_raw_test.go index b06a7b5..b9eef89 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -92,10 +92,10 @@ func TestRAWInputNoKeepAlive(t *testing.T) { t.Fatal(err) } origin := &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("a")) - w.Write([]byte("b")) - }), + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("a")) + w.Write([]byte("b")) + }), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, } @@ -105,7 +105,7 @@ func TestRAWInputNoKeepAlive(t *testing.T) { originAddr := listener.Addr().String() - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http") defer input.Close() output := NewTestOutput(func(data []byte) { From 8e37136b8e941afb1df5ea351cc4bba03e17cdbd Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 10 Aug 2016 14:06:42 +0300 Subject: [PATCH 48/79] Fix test (one more time) --- output_http.go | 4 +--- output_http_test.go | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/output_http.go b/output_http.go index c57c8c5..cf880a5 100644 --- a/output_http.go +++ b/output_http.go @@ -148,6 +148,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { func (o *HTTPOutput) workerMaster() { for { newWorkers := <-o.needWorker + atomic.AddInt64(&o.activeWorkers, int64(newWorkers)) for i := 0; i < newWorkers; i++ { go o.startWorker() } @@ -171,7 +172,6 @@ func (o *HTTPOutput) sessionWorkerMaster() { if !ok { atomic.AddInt64(&o.activeWorkers, 1) - worker = newHTTPWorker(o, nil) o.workerSessions[sessionID] = worker } @@ -201,8 +201,6 @@ func (o *HTTPOutput) startWorker() { ResponseBufferSize: o.config.BufferSize, }) - atomic.AddInt64(&o.activeWorkers, 1) - for { select { case data := <-o.queue: diff --git a/output_http_test.go b/output_http_test.go index a3324a2..cf64ad3 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -57,12 +57,11 @@ func TestHTTPOutput(t *testing.T) { input.EmitGET() } - wg.Wait() - - if output.(*HTTPOutput).activeWorkers != 200 { + if output.(*HTTPOutput).activeWorkers < 200 { t.Error("Should create workers for each request", output.(*HTTPOutput).activeWorkers) } + wg.Wait() close(quit) Settings.modifierConfig = HTTPModifierConfig{} From a12357b49737165bca25edfc1365c92902325bac Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 10 Aug 2016 14:08:59 +0300 Subject: [PATCH 49/79] Fix Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6809af0..335771e 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ FADDR = ":8000" release: release-x64 release-mac release-x64: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor release-x86: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor release-mac: go build -o gor $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_PRO_mac.tar.gz gor && rm gor From 75b31fa701bfebc54cdba3c794f0441013786ade Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 10 Aug 2016 15:01:06 +0300 Subject: [PATCH 50/79] Fix make file --- Makefile | 4 ++-- s3/index.html | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 335771e..67cb277 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ FADDR = ":8000" release: release-x64 release-mac release-x64: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_PRO_x64.tar.gz gor && rm gor release-x86: - docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_PRO_x86.tar.gz gor && rm gor release-mac: go build -o gor $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_PRO_mac.tar.gz gor && rm gor diff --git a/s3/index.html b/s3/index.html index 5557576..3502792 100644 --- a/s3/index.html +++ b/s3/index.html @@ -38,6 +38,13 @@ ul {

Gor PRO releases

See releases page on GitHub for changelog

+

v0.15.0

+ +

v0.14.1

  • gor_v0.14.1_PRO_x64.tar.gz - Linux x64
    • From 082106dfd8d92be824e1da38dd5db8982fa1a5bf Mon Sep 17 00:00:00 2001 From: Mourjo Sen Date: Sat, 20 Aug 2016 22:25:45 +0530 Subject: [PATCH 51/79] Fix Java middleware example (#358) * Add encoding/decoding HTTP msgs to Java middleware example. * Add clojure middleware example. --- examples/middleware/echo.clj | 53 +++++++++++++++++++++++++++++++++++ examples/middleware/echo.java | 29 ++++++++++++++++--- 2 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 examples/middleware/echo.clj diff --git a/examples/middleware/echo.clj b/examples/middleware/echo.clj new file mode 100644 index 0000000..1757235 --- /dev/null +++ b/examples/middleware/echo.clj @@ -0,0 +1,53 @@ +(ns echo.core + (:gen-class) + (:require [clojure.string :as cs] + [clojure.java.io :as io]) + (:import org.apache.commons.codec.binary.Hex + java.io.BufferedReader + java.io.IOException + java.io.InputStreamReader)) + + +(defn transform-http-msg + "Function that transforms/filters the incoming HTTP messages." + [headers body] + ;; do actual transformations here + [headers body]) + + +(defn decode-hex-string + "Decode an Hex-encoded string." + [s] + (String. (Hex/decodeHex (.toCharArray s)))) + + +(defn encode-hex-string + "Encode a string to a hex-encoded string." + [^String s] + (String. (Hex/encodeHex (.getBytes s)))) + + +(defn -main + [& args] + (let [br (BufferedReader. (InputStreamReader. System/in))] + (try + (loop [hex-line (.readLine br)] + (let [decoded-req (decode-hex-string hex-line) + + ;; empty line separates headers from body + http-request (partition-by empty? (cs/split-lines decoded-req)) + headers (first http-request) + + ;; HTTP messages can contain no body: + body (when (= 3 (count http-request)) (last http-request)) + [new-headers new-body] (transform-http-msg headers body)] + + (println (encode-hex-string (str (cs/join "\n" headers) + (when body + (str "\n\n" + (cs/join "\n" body))))))) + (when-let [line (.readLine br)] + (recur line))) + (catch IOException e nil)))) + + diff --git a/examples/middleware/echo.java b/examples/middleware/echo.java index 8464819..ffa885c 100644 --- a/examples/middleware/echo.java +++ b/examples/middleware/echo.java @@ -2,8 +2,25 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -public class echo { - public static void main(String[] args) { +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; + + +public class Echo { + public static String decodeHexString(String s) throws DecoderException { + return new String(Hex.decodeHex(s.toCharArray())); + } + + public static String encodeHexString(String s) { + return new String(Hex.encodeHex(s.getBytes())); + } + + public static String transformHTTPMessage(String req) { + // do actual transformations here + return req; + } + + public static void main(String[] args) throws DecoderException { if(args != null){ for(String arg : args){ System.out.println(arg); @@ -17,11 +34,15 @@ public class echo { try { while ((line = stdin.readLine()) != null) { + String decodedLine = decodeHexString(line); - System.out.println(line); + String transformedLine = transformHTTPMessage(decodedLine); + + String encodedLine = encodeHexString(transformedLine); + System.out.println(encodedLine); } } catch (IOException e) { } } -} \ No newline at end of file +} From 647f7022cfa0fa4d0d8a97fb8109060a9e44eda3 Mon Sep 17 00:00:00 2001 From: Guillaume Gelin Date: Tue, 23 Aug 2016 18:29:27 +0200 Subject: [PATCH 52/79] Add google/gopacket in vendor/ (#360) --- .gitmodules | 3 +++ vendor/github.com/google/gopacket | 1 + 2 files changed, 4 insertions(+) create mode 160000 vendor/github.com/google/gopacket diff --git a/.gitmodules b/.gitmodules index 1c1686b..a8037c9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "vendor/github.com/bmizerany/assert"] path = vendor/github.com/bmizerany/assert url = https://github.com/bmizerany/assert +[submodule "vendor/github.com/google/gopacket"] + path = vendor/github.com/google/gopacket + url = https://github.com/google/gopacket diff --git a/vendor/github.com/google/gopacket b/vendor/github.com/google/gopacket new file mode 160000 index 0000000..c4d6479 --- /dev/null +++ b/vendor/github.com/google/gopacket @@ -0,0 +1 @@ +Subproject commit c4d647984d4671d9dd91770dae77923a0b48b389 From fc0361dcdcbc41541ba6378f63197e37d8a38939 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 31 Aug 2016 16:09:39 +0300 Subject: [PATCH 53/79] Fix handling of connection: close for POST requests --- raw_socket_listener/listener.go | 27 ++++++++++++++---- raw_socket_listener/tcp_message.go | 45 ++++++++++++++++-------------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index f85597b..8e035e2 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -115,8 +115,6 @@ func NewListener(addr string, port string, engine int, trackResponse bool, expir // Special case for testing if l.port != 0 { switch engine { - case EngineRawSocket: - go l.readRAWSocket() case EnginePcap: go l.readPcap() case EnginePcapFile: @@ -559,7 +557,14 @@ func (t *Listener) readPcapFile() { if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil { tcp, _ := tcpLayer.(*layers.TCP) data = append(tcp.LayerContents(), tcp.LayerPayload()...) - copy(data[2:4], []byte{0, 1}) + + if tcp.SrcPort >= 32768 && tcp.SrcPort <= 61000 { + copy(data[0:2], []byte{0, 0}) + copy(data[2:4], []byte{0, 1}) + } else { + copy(data[0:2], []byte{0, 1}) + copy(data[2:4], []byte{0, 0}) + } } else { continue } @@ -576,10 +581,11 @@ func (t *Listener) readPcapFile() { } dataOffset := (data[12] & 0xF0) >> 4 + isFIN := data[13]&0x01 != 0 // We need only packets with data inside // Check that the buffer is larger than the size of the TCP header - if len(data) <= int(dataOffset*4) { + if len(data) <= int(dataOffset*4) && !isFIN { continue } @@ -663,8 +669,6 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } }() - // log.Println("Processing packet:", packet.Ack, packet.Seq, packet.ID) - var message *TCPMessage isIncoming := packet.DestPort == t.port @@ -693,6 +697,14 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { packet.UpdateAck(parentAck) } + if isIncoming && packet.IsFIN { + if ma, ok := t.respAliases[packet.Seq]; ok { + if ma.packets[0].SrcPort == packet.SrcPort { + packet.UpdateAck(ma.Ack) + } + } + } + if alias, ok := t.ackAliases[packet.Ack]; ok { packet.UpdateAck(alias) } @@ -764,8 +776,11 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { // If message contains only single packet immediately dispatch it if message.complete { + // log.Println("COMPLETE!", isIncoming, message) if isIncoming { if t.trackResponse { + // log.Println("Found response!", message.ResponseID, t.messages) + if resp, ok := t.messages[message.ResponseID]; ok { if resp.complete { t.dispatchMessage(resp) diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 7a5185c..5cb566e 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -352,6 +352,26 @@ func (t *TCPMessage) updateBodyType() { return } + var lengthB, encB, connB []byte + + proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{ + if proto.HeadersEqual(header, []byte("Content-Length")) { + lengthB = value + return false + } + + if proto.HeadersEqual(header, []byte("Transfer-Encoding")) { + encB = value + return false + } + + if proto.HeadersEqual(header, []byte("Connection")) { + connB = value + } + + return true + }) + switch t.methodType { case httpMethodNotFound: return @@ -359,27 +379,6 @@ func (t *TCPMessage) updateBodyType() { t.bodyType = httpBodyEmpty return case httpMethodWithBody: - var lengthB, encB, connB []byte - - proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{ - if proto.HeadersEqual(header, []byte("Content-Length")) { - lengthB = value - return false - } - - if proto.HeadersEqual(header, []byte("Transfer-Encoding")) { - encB = value - return false - } - - if proto.HeadersEqual(header, []byte("Connection")) { - connB = value - return false - } - - return true - }) - if len(lengthB) > 0 { t.contentLength, _ = strconv.Atoi(string(lengthB)) @@ -461,6 +460,10 @@ func (t *TCPMessage) setAssocMessage(m *TCPMessage) { // UpdateResponseAck should be called after packet is added func (t *TCPMessage) UpdateResponseAck() uint32 { lastPacket := t.packets[len(t.packets)-1] + if lastPacket.IsFIN && len(t.packets) > 1 { + lastPacket = t.packets[len(t.packets)-2] + } + respAck := lastPacket.Seq + uint32(len(lastPacket.Data)) if t.ResponseAck != respAck { From a2f5e4337576f2fc2c5a2e6cbb06a14c155573eb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 31 Aug 2016 16:27:07 +0300 Subject: [PATCH 54/79] Fix replayed response header --- output_http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output_http.go b/output_http.go index d442b83..fc909a6 100644 --- a/output_http.go +++ b/output_http.go @@ -179,7 +179,7 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { Debug("[OUTPUT-HTTP] Received response:", string(resp.payload)) - header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime) + header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt) copy(data[0:len(header)], header) copy(data[len(header):], resp.payload) From 1674b58707c3a94243ac54351257f70409875940 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 31 Aug 2016 17:27:18 +0300 Subject: [PATCH 55/79] Fix basic http auth --- http_client.go | 10 ++++++++++ http_client_test.go | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/http_client.go b/http_client.go index 361ecc6..c4ec8f2 100644 --- a/http_client.go +++ b/http_client.go @@ -3,6 +3,7 @@ package main import ( "bytes" "crypto/tls" + "encoding/base64" "io" "log" "net" @@ -44,6 +45,7 @@ type HTTPClient struct { baseURL string scheme string host string + auth string conn net.Conn respBuf []byte config *HTTPClientConfig @@ -79,6 +81,10 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { client.respBuf = make([]byte, config.ResponseBufferSize) client.config = config + if u.User != nil { + client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String())) + } + return client } @@ -164,6 +170,10 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host)) } + if c.auth != "" { + data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth)) + } + if c.config.Debug { Debug("[HTTPClient] Sending:", string(data)) } diff --git a/http_client_test.go b/http_client_test.go index 79c71e7..00f9db3 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "net/http/httputil" _ "reflect" + "strings" "sync" "testing" "time" @@ -92,7 +93,7 @@ func TestHTTPClientResonseByClose(t *testing.T) { payload := []byte("GET / HTTP/1.1\r\n\r\n") ln, _ := net.Listen("tcp", ":0") - go func(){ + go func() { for { conn, _ := ln.Accept() buf := make([]byte, 4096) @@ -356,6 +357,45 @@ func TestHTTPClientRedirectLimit(t *testing.T) { wg.Wait() } +func TestHTTPClientBasicAuth(t *testing.T) { + wg := new(sync.WaitGroup) + wg.Add(2) + + GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, pass, _ := r.BasicAuth() + + if user != "user" || pass != "pass" { + http.Error(w, "Unauthorized.", 401) + wg.Done() + return + } + + wg.Done() + })) + defer server.Close() + + client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false}) + resp, _ := client.Send(GETPayload) + client.Disconnect() + + if !bytes.Equal(proto.Status(resp), []byte("401")) { + t.Error("Should return unauthorized error", string(resp)) + } + + authUrl := strings.Replace(server.URL, "http://", "http://user:pass@", -1) + client = NewHTTPClient(authUrl, &HTTPClientConfig{Debug: false}) + resp, _ = client.Send(GETPayload) + client.Disconnect() + + if !bytes.Equal(proto.Status(resp), []byte("200")) { + t.Error("Should return proper response", string(resp)) + } + + wg.Wait() +} + func TestHTTPClientHandleHTTP10(t *testing.T) { wg := new(sync.WaitGroup) From 967c380dc3ca1a96c6cbabd6296b0656a6546016 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 31 Aug 2016 17:27:45 +0300 Subject: [PATCH 56/79] go fmt --- elasticsearch.go | 2 +- gor.go | 2 +- input_raw_test.go | 8 ++++---- output_null.go | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/elasticsearch.go b/elasticsearch.go index 2ff4ab9..a626853 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -2,8 +2,8 @@ package main import ( "encoding/json" - "github.com/mattbaird/elastigo/lib" "github.com/buger/gor/proto" + "github.com/mattbaird/elastigo/lib" "log" "regexp" "time" diff --git a/gor.go b/gor.go index 085eb98..b9cf1f8 100644 --- a/gor.go +++ b/gor.go @@ -132,4 +132,4 @@ func profileMEM(memprofile string) { f.Close() }) } -} \ No newline at end of file +} diff --git a/input_raw_test.go b/input_raw_test.go index a40b0fa..9ad8065 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -92,10 +92,10 @@ func TestRAWInputNoKeepAlive(t *testing.T) { t.Fatal(err) } origin := &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("a")) - w.Write([]byte("b")) - }), + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("a")) + w.Write([]byte("b")) + }), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, } diff --git a/output_null.go b/output_null.go index 14867a2..4b756db 100644 --- a/output_null.go +++ b/output_null.go @@ -6,13 +6,13 @@ type NullOutput struct { // NullOutput constructor for NullOutput func NewNullOutput() (o *NullOutput) { - return new(NullOutput) + return new(NullOutput) } func (o *NullOutput) Write(data []byte) (int, error) { - return len(data), nil + return len(data), nil } func (o *NullOutput) String() string { - return "Null Output" + return "Null Output" } From 7f7cb7c4390bdce715a2badbba187c2ae459f2ab Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 31 Aug 2016 18:39:56 +0300 Subject: [PATCH 57/79] Fix tests --- output_http_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output_http_test.go b/output_http_test.go index cf64ad3..154e5c5 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -57,7 +57,7 @@ func TestHTTPOutput(t *testing.T) { input.EmitGET() } - if output.(*HTTPOutput).activeWorkers < 200 { + if output.(*HTTPOutput).activeWorkers < 50 { t.Error("Should create workers for each request", output.(*HTTPOutput).activeWorkers) } From 0831476beb90bb03e8d2648dfcc8d2f63f1486cb Mon Sep 17 00:00:00 2001 From: "Huu Khiem (Mark)" Date: Tue, 27 Sep 2016 15:40:26 +0800 Subject: [PATCH 58/79] Set default value for output-http-timeout at declaration (#371) The actual default is 5s, set in the call of NewHTTPClient, even if we use `--output-http--timeout=0`. It's clearer to user if this value id declared upfront. --- http_client.go | 4 ---- settings.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/http_client.go b/http_client.go index c4ec8f2..0d4df18 100644 --- a/http_client.go +++ b/http_client.go @@ -64,10 +64,6 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { } } - if config.Timeout.Nanoseconds() == 0 { - config.Timeout = 5 * time.Second - } - config.ConnectionTimeout = config.Timeout if config.ResponseBufferSize == 0 { diff --git a/settings.go b/settings.go index 04828e7..b71bf56 100644 --- a/settings.go +++ b/settings.go @@ -118,7 +118,7 @@ func init() { flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") - flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") + flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5 * time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.") From 47db326b0f0e5dd30d3f432821a4cf8e27517211 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Oct 2016 18:56:27 +0300 Subject: [PATCH 59/79] Fix truncated tcp check --- .gitignore | 3 +++ raw_socket_listener/listener.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4c1635a..6bc8d66 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.bin *.gz +*.zip *.class @@ -17,3 +18,5 @@ gor *.mprof *.pcap + +.DS_Store diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 8e035e2..cecf979 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -465,7 +465,7 @@ func (t *Listener) readPcap() { } // Truncated TCP info - if len(data) < 13 { + if len(data) <= 13 { continue } From 22466d402818abdc935bfe0149b7569cee06df6e Mon Sep 17 00:00:00 2001 From: manjeshnilange Date: Tue, 25 Oct 2016 12:06:43 -0700 Subject: [PATCH 60/79] =?UTF-8?q?Adding=20ability=20to=20output=20http=20c?= =?UTF-8?q?lient=20to=20close=20connection=20on=20specific=20=E2=80=A6=20(?= =?UTF-8?q?#375)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adding ability to output http client to close connection on specific response status * Addressed review comments --- http_client.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/http_client.go b/http_client.go index 0d4df18..b14242a 100644 --- a/http_client.go +++ b/http_client.go @@ -318,6 +318,11 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { } } + if bytes.Equal(proto.Status(payload), []byte("400")) { + c.Disconnect() + Debug("[HTTPClient] Closed connection on 400 response") + } + c.redirectsCount = 0 return payload, err From 76fb91966b77f722f5f155ffbcaa8bc6d1d1964b Mon Sep 17 00:00:00 2001 From: Jonathan Cremin Date: Thu, 27 Oct 2016 18:44:54 +0100 Subject: [PATCH 61/79] Treat PATCH as a HTTP verb --- proto/proto.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/proto.go b/proto/proto.go index 0bdb819..53b627c 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -443,7 +443,7 @@ func Status(payload []byte) []byte { } var httpMethods []string = []string{ - "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN" /* custom methods */, "BAN", "PURG", + "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN", "PURG", } func IsHTTPPayload(payload []byte) bool { From e3a9a9b2c20b23448036fb566890e38c1e2fdd0b Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Mon, 31 Oct 2016 19:04:05 +0300 Subject: [PATCH 62/79] Add kafka output --- emitter.go | 2 +- output_kafka.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ plugins.go | 2 ++ settings.go | 6 ++++- 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 output_kafka.go diff --git a/emitter.go b/emitter.go index 1fc2a83..f085a2e 100644 --- a/emitter.go +++ b/emitter.go @@ -15,7 +15,7 @@ func Start(stop chan int) { middleware.ReadFrom(in) } - // We going only to read responses, so using same ReadFrom method + // We are going only to read responses, so using same ReadFrom method for _, out := range Plugins.Outputs { if r, ok := out.(io.Reader); ok { middleware.ReadFrom(r) diff --git a/output_kafka.go b/output_kafka.go new file mode 100644 index 0000000..cf2ea1f --- /dev/null +++ b/output_kafka.go @@ -0,0 +1,68 @@ +package main + +import ( + "github.com/Shopify/sarama" + "log" + "strings" + "time" +) + +// KafkaConfig should contains required information to +// build producers. +type KafkaConfig struct { + zookeeper string + topic string +} + +// KafkaOutput should make producer client. +type KafkaOutput struct { + address string + config *KafkaConfig + producer sarama.AsyncProducer +} + +// NewKafkaOutput creates instance of kafka producer client. +func NewKafkaOutput(address string, config *KafkaConfig) *KafkaOutput { + c := sarama.NewConfig() + c.Producer.RequiredAcks = sarama.WaitForLocal + c.Producer.Compression = sarama.CompressionSnappy + c.Producer.Flush.Frequency = 500 * time.Millisecond + + brokerList := strings.Split(config.zookeeper, ",") + + producer, err := sarama.NewAsyncProducer(brokerList, c) + if err != nil { + log.Fatalln("Failed to start Sarama(Kafka) producer:", err) + } + + o := &KafkaOutput{ + address: address, + config: config, + producer: producer, + } + + // Start infinite loop for tracking errors for kafka producer. + go o.ErrorHandler() + + return o +} + +// ErrorHandler should receive errors +func (o *KafkaOutput) ErrorHandler() { + for err := range o.producer.Errors() { + log.Println("Failed to write access log entry:", err) + } +} + +func (o *KafkaOutput) Write(data []byte) (n int, err error) { + buf := make(sarama.ByteEncoder, len(data)) + copy(buf, data) + + o.producer.Input() <- &sarama.ProducerMessage{ + Topic: o.config.topic, + Key: sarama.StringEncoder(o.address), + Value: buf, + } + + return len(data), nil +} diff --git a/plugins.go b/plugins.go index 1a0f6db..1d11971 100644 --- a/plugins.go +++ b/plugins.go @@ -142,4 +142,6 @@ func InitPlugins() { for _, options := range Settings.outputHTTP { registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig) } + + registerPlugin(NewKafkaOutput, &Settings.outputKafkaConfig) } diff --git a/settings.go b/settings.go index b71bf56..0b4973b 100644 --- a/settings.go +++ b/settings.go @@ -58,6 +58,8 @@ type AppSettings struct { outputHTTPConfig HTTPOutputConfig modifierConfig HTTPModifierConfig + + outputKafkaConfig KafkaConfig } // Settings holds Gor configuration @@ -118,13 +120,15 @@ func init() { flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") - flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5 * time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") + flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.") flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.") flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") + flag.StringVar(&Settings.outputKafkaConfig.zookeeper, "output-kafka-zookeeper", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-zookeeper '192.168.0.1:2181,192.168.0.2:2181'") + flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'") flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead") From 9b96dc20df2dd885c1653c00d5252b1ff25c6c3d Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Mon, 31 Oct 2016 19:29:59 +0300 Subject: [PATCH 63/79] Change Dockerfile because of issue on build echo.java --- Dockerfile | 4 +++- examples/middleware/echo.java | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98fb7a4..f3d1874 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,5 +18,7 @@ RUN go get -u github.com/golang/lint/golint WORKDIR /go/src/github.com/buger/gor/ ADD . /go/src/github.com/buger/gor/ -RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java +RUN wget http://archive.apache.org/dist/commons/io/binaries/commons-io-2.4-bin.tar.gz && tar xzf commons-io-2.4-bin.tar.gz && cd commons-io-2.4 && mv commons-io-2.4.jar /tmp/ +RUN wget http://archive.apache.org/dist/commons/codec/binaries/commons-codec-1.9-bin.tar.gz && tar xzf commons-codec-1.9-bin.tar.gz +RUN javac -cp commons-io-2.4/commons-io-2.4.jar -cp commons-codec-1.9/commons-codec-1.9.jar ./examples/middleware/echo.java RUN go get \ No newline at end of file diff --git a/examples/middleware/echo.java b/examples/middleware/echo.java index ffa885c..9fba333 100644 --- a/examples/middleware/echo.java +++ b/examples/middleware/echo.java @@ -6,19 +6,19 @@ import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; -public class Echo { - public static String decodeHexString(String s) throws DecoderException { - return new String(Hex.decodeHex(s.toCharArray())); - } +class Echo { + public static String decodeHexString(String s) throws DecoderException { + return new String(Hex.decodeHex(s.toCharArray())); + } - public static String encodeHexString(String s) { - return new String(Hex.encodeHex(s.getBytes())); - } + public static String encodeHexString(String s) { + return new String(Hex.encodeHex(s.getBytes())); + } - public static String transformHTTPMessage(String req) { - // do actual transformations here - return req; - } + public static String transformHTTPMessage(String req) { + // do actual transformations here + return req; + } public static void main(String[] args) throws DecoderException { if(args != null){ @@ -29,7 +29,7 @@ public class Echo { } BufferedReader stdin = new BufferedReader(new InputStreamReader( - System.in)); + System.in)); String line = null; try { From 41d3ce48a3d8e372935f09cfcfdf39f8d2259bb3 Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Tue, 1 Nov 2016 17:51:15 +0300 Subject: [PATCH 64/79] Add json message output for kafka --- elasticsearch.go | 8 ++++--- examples/server.go | 15 ++++++++++++ output_kafka.go | 58 +++++++++++++++++++++++++++++++++++----------- plugins.go | 2 +- settings.go | 3 ++- 5 files changed, 67 insertions(+), 19 deletions(-) create mode 100644 examples/server.go diff --git a/elasticsearch.go b/elasticsearch.go index a626853..2635b89 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -84,9 +84,11 @@ func (p *ESPlugin) Init(URI string) { p.done = make(chan bool) p.indexor.Start() - // Only start the ErrorHandler goroutine when in verbose mode - // no need to burn ressources otherwise - go p.ErrorHandler() + if Settings.verbose { + // Only start the ErrorHandler goroutine when in verbose mode + // no need to burn ressources otherwise + go p.ErrorHandler() + } log.Println("Initialized Elasticsearch Plugin") return diff --git a/examples/server.go b/examples/server.go new file mode 100644 index 0000000..8be30c6 --- /dev/null +++ b/examples/server.go @@ -0,0 +1,15 @@ +package main + +import ( + "io" + "net/http" +) + +func hello(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "Hello world!") +} + +func main() { + http.HandleFunc("/", hello) + http.ListenAndServe(":8000", nil) +} diff --git a/output_kafka.go b/output_kafka.go index cf2ea1f..9af4ca0 100644 --- a/output_kafka.go +++ b/output_kafka.go @@ -1,7 +1,10 @@ package main import ( + "encoding/json" "github.com/Shopify/sarama" + "github.com/buger/gor/proto" + "io" "log" "strings" "time" @@ -10,25 +13,41 @@ import ( // KafkaConfig should contains required information to // build producers. type KafkaConfig struct { - zookeeper string - topic string + host string + topic string } // KafkaOutput should make producer client. type KafkaOutput struct { - address string config *KafkaConfig producer sarama.AsyncProducer } +// KafkaMessage should contains catched request information that should be +// passed as Json to Apache Kafka. +type KafkaMessage struct { + ReqURL string `json:"Req_URL"` + ReqMethod string `json:"Req_Method"` + ReqUserAgent string `json:"Req_User-Agent"` + ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"` + ReqAccept string `json:"Req_Accept,omitempty"` + ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"` + ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"` + ReqConnection string `json:"Req_Connection,omitempty"` + ReqCookies string `json:"Req_Cookies,omitempty"` +} + +// KafkaOutputFrequency in milliseconds +const KafkaOutputFrequency = 500 + // NewKafkaOutput creates instance of kafka producer client. -func NewKafkaOutput(address string, config *KafkaConfig) *KafkaOutput { +func NewKafkaOutput(address string, config *KafkaConfig) io.Writer { c := sarama.NewConfig() c.Producer.RequiredAcks = sarama.WaitForLocal c.Producer.Compression = sarama.CompressionSnappy - c.Producer.Flush.Frequency = 500 * time.Millisecond + c.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond - brokerList := strings.Split(config.zookeeper, ",") + brokerList := strings.Split(config.host, ",") producer, err := sarama.NewAsyncProducer(brokerList, c) if err != nil { @@ -36,13 +55,14 @@ func NewKafkaOutput(address string, config *KafkaConfig) *KafkaOutput { } o := &KafkaOutput{ - address: address, config: config, producer: producer, } - // Start infinite loop for tracking errors for kafka producer. - go o.ErrorHandler() + if Settings.verbose { + // Start infinite loop for tracking errors for kafka producer. + go o.ErrorHandler() + } return o } @@ -55,14 +75,24 @@ func (o *KafkaOutput) ErrorHandler() { } func (o *KafkaOutput) Write(data []byte) (n int, err error) { - buf := make(sarama.ByteEncoder, len(data)) - copy(buf, data) + kafkaMessage := KafkaMessage{ + ReqURL: string(proto.Path(data)), + ReqMethod: string(proto.Method(data)), + ReqUserAgent: string(proto.Header(data, []byte("User-Agent"))), + ReqAcceptLanguage: string(proto.Header(data, []byte("Accept-Language"))), + ReqAccept: string(proto.Header(data, []byte("Accept"))), + ReqAcceptEncoding: string(proto.Header(data, []byte("Accept-Encoding"))), + ReqIfModifiedSince: string(proto.Header(data, []byte("If-Modified-Since"))), + ReqConnection: string(proto.Header(data, []byte("Connection"))), + ReqCookies: string(proto.Header(data, []byte("Cookie"))), + } + jsonMessage, _ := json.Marshal(&kafkaMessage) + message := sarama.StringEncoder(jsonMessage) o.producer.Input() <- &sarama.ProducerMessage{ Topic: o.config.topic, - Key: sarama.StringEncoder(o.address), - Value: buf, + Value: message, } - return len(data), nil + return len(message), nil } diff --git a/plugins.go b/plugins.go index 1d11971..a625f9b 100644 --- a/plugins.go +++ b/plugins.go @@ -143,5 +143,5 @@ func InitPlugins() { registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig) } - registerPlugin(NewKafkaOutput, &Settings.outputKafkaConfig) + registerPlugin(NewKafkaOutput, "", &Settings.outputKafkaConfig) } diff --git a/settings.go b/settings.go index 0b4973b..ff417ca 100644 --- a/settings.go +++ b/settings.go @@ -127,7 +127,8 @@ func init() { flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.") flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") - flag.StringVar(&Settings.outputKafkaConfig.zookeeper, "output-kafka-zookeeper", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-zookeeper '192.168.0.1:2181,192.168.0.2:2181'") + + flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:2181,192.168.0.2:2181'") flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'") flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") From d424167f0496242ec7246ba6b6da3e1d8f9b9c64 Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Tue, 1 Nov 2016 17:55:04 +0300 Subject: [PATCH 65/79] Change example of port for kafka host --- settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.go b/settings.go index ff417ca..ac62f7c 100644 --- a/settings.go +++ b/settings.go @@ -128,7 +128,7 @@ func init() { flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") - flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:2181,192.168.0.2:2181'") + flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'") flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'") flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") From 0a038575f3d8eeca3158979a4303d4fc4fbeaa4f Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Wed, 2 Nov 2016 00:03:12 +0300 Subject: [PATCH 66/79] Add all headers and properly passed body --- output_kafka.go | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/output_kafka.go b/output_kafka.go index 9af4ca0..4ebfc4c 100644 --- a/output_kafka.go +++ b/output_kafka.go @@ -26,15 +26,10 @@ type KafkaOutput struct { // KafkaMessage should contains catched request information that should be // passed as Json to Apache Kafka. type KafkaMessage struct { - ReqURL string `json:"Req_URL"` - ReqMethod string `json:"Req_Method"` - ReqUserAgent string `json:"Req_User-Agent"` - ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"` - ReqAccept string `json:"Req_Accept,omitempty"` - ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"` - ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"` - ReqConnection string `json:"Req_Connection,omitempty"` - ReqCookies string `json:"Req_Cookies,omitempty"` + ReqURL string `json:"Req_URL"` + ReqMethod string `json:"Req_Method"` + ReqBody string `json:"Req_Body,omitempty"` + ReqHeaders map[string]string `json:"Req_Headers,omitempty"` } // KafkaOutputFrequency in milliseconds @@ -75,16 +70,19 @@ func (o *KafkaOutput) ErrorHandler() { } func (o *KafkaOutput) Write(data []byte) (n int, err error) { + headers := make(map[string]string) + proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool { + headers[string(header)] = string(value) + return true + }) + + req := payloadBody(data) + kafkaMessage := KafkaMessage{ - ReqURL: string(proto.Path(data)), - ReqMethod: string(proto.Method(data)), - ReqUserAgent: string(proto.Header(data, []byte("User-Agent"))), - ReqAcceptLanguage: string(proto.Header(data, []byte("Accept-Language"))), - ReqAccept: string(proto.Header(data, []byte("Accept"))), - ReqAcceptEncoding: string(proto.Header(data, []byte("Accept-Encoding"))), - ReqIfModifiedSince: string(proto.Header(data, []byte("If-Modified-Since"))), - ReqConnection: string(proto.Header(data, []byte("Connection"))), - ReqCookies: string(proto.Header(data, []byte("Cookie"))), + ReqURL: string(proto.Path(req)), + ReqMethod: string(proto.Method(req)), + ReqBody: string(proto.Body(req)), + ReqHeaders: headers, } jsonMessage, _ := json.Marshal(&kafkaMessage) message := sarama.StringEncoder(jsonMessage) From 40f7facc3328b6c9ba4dd9ab092119ed437cf937 Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Wed, 2 Nov 2016 15:34:42 +0300 Subject: [PATCH 67/79] User-Agent could contains ':' inside of value. --- proto/proto.go | 7 ++++++- proto/proto_test.go | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/proto/proto.go b/proto/proto.go index 0bdb819..c0bdd50 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -190,6 +190,7 @@ func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) i := 0 pIdx := 0 lineBreaks := 0 + newLineBreak := true for { if len(payloads)-1 < pIdx { @@ -206,6 +207,7 @@ func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) switch p[i] { case '\r', '\n': + newLineBreak = true lineBreaks++ // End of headers @@ -254,7 +256,10 @@ func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) hS = [2]int{-1, -1} hE = [2]int{-1, -1} case ':': - hE = [2]int{pIdx, i} + if newLineBreak { + hE = [2]int{pIdx, i} + newLineBreak = false + } default: lineBreaks = 0 diff --git a/proto/proto_test.go b/proto/proto_test.go index 32da8c9..7d25a38 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -139,11 +139,33 @@ func TestParseHeaders(t *testing.T) { "Host": "www.w3.org", "User-Agent": "Chrome", } + if !reflect.DeepEqual(headers, expected) { t.Error("Headers do not properly parsed", headers) } } +func TestParseHeadersWithComplexUserAgent(t *testing.T) { + // User-Agent could contain inside ':' + // Parser should wait for \r\n + payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("Fake-Header: asda")} + + headers := make(map[string]string) + + ParseHeaders(payload, func(header []byte, value []byte) bool { + headers[string(header)] = string(value) + return true + }) + + expected := map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", + } + + if expected["User-Agent"] != headers["User-Agent"] { + t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"]) + } +} + func TestHeaderEquals(t *testing.T) { tests := []struct { h1 string From 638ddfb144b275c3adde9096229c4e6cdcd3d226 Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Wed, 2 Nov 2016 17:50:45 +0300 Subject: [PATCH 68/79] Add one more test case to cover ':' inside of header value --- proto/proto_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/proto/proto_test.go b/proto/proto_test.go index 7d25a38..800799b 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -166,6 +166,37 @@ func TestParseHeadersWithComplexUserAgent(t *testing.T) { } } +func TestParseHeadersWithOrigin(t *testing.T) { + // User-Agent could contain inside ':' + // Parser should wait for \r\n + payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nReferrer: http://127.0.0.1:3000\r\nOrigi"), []byte("n: https://www.example.com\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("in:https://www.example.com\r\n\r\n"), []byte("Fake-Header: asda")} + + headers := make(map[string]string) + + ParseHeaders(payload, func(header []byte, value []byte) bool { + headers[string(header)] = string(value) + return true + }) + + expected := map[string]string{ + "Origin": "https://www.example.com", + "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Referrer": "http://127.0.0.1:3000", + } + + if expected["Referrer"] != headers["Referrer"] { + t.Errorf("Header 'Referrer' expected '%s' and parsed: '%s'", expected["Referrer"], headers["Referrer"]) + } + + if expected["Origin"] != headers["Origin"] { + t.Errorf("Header 'Origin' expected '%s' and parsed: '%s'", expected["Origin"], headers["Origin"]) + } + + if expected["User-Agent"] != headers["User-Agent"] { + t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"]) + } +} + func TestHeaderEquals(t *testing.T) { tests := []struct { h1 string From f90dcc763b598bf4060af0fe05f13d9caafa4d57 Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Tue, 8 Nov 2016 22:21:35 +0300 Subject: [PATCH 69/79] Remove server.go because of having 'gor file-server :8080' --- examples/server.go | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 examples/server.go diff --git a/examples/server.go b/examples/server.go deleted file mode 100644 index 8be30c6..0000000 --- a/examples/server.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - "io" - "net/http" -) - -func hello(w http.ResponseWriter, r *http.Request) { - io.WriteString(w, "Hello world!") -} - -func main() { - http.HandleFunc("/", hello) - http.ListenAndServe(":8000", nil) -} From a1174a159c42361b0b151280631598d51548edf0 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 14 Nov 2016 17:47:30 +0300 Subject: [PATCH 70/79] Do not add port to Host header for https --- http_client.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/http_client.go b/http_client.go index b14242a..a47e850 100644 --- a/http_client.go +++ b/http_client.go @@ -58,11 +58,6 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { } u, _ := url.Parse(baseURL) - if !strings.Contains(u.Host, ":") { - if u.Scheme != "http" { - u.Host += ":" + defaultPorts[u.Scheme] - } - } config.ConnectionTimeout = config.Timeout @@ -88,7 +83,7 @@ func (c *HTTPClient) Connect() (err error) { c.Disconnect() if !strings.Contains(c.host, ":") { - c.conn, err = net.DialTimeout("tcp", c.host+":80", c.config.ConnectionTimeout) + c.conn, err = net.DialTimeout("tcp", c.host + ":" + defaultPorts[c.scheme], c.config.ConnectionTimeout) } else { c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout) } From 963f7a73d7697d946314b7dd0d58310dcfaa6395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Solano=20G=C3=B3mez?= Date: Wed, 23 Nov 2016 09:55:54 -0500 Subject: [PATCH 71/79] Add basic SNI support By adding the server name as part of the TLS client configuration, gor will now support connecting to hosts that require SNI (such as Amazon API Gateway). --- http_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http_client.go b/http_client.go index a47e850..8ee42e4 100644 --- a/http_client.go +++ b/http_client.go @@ -89,7 +89,7 @@ func (c *HTTPClient) Connect() (err error) { } if c.scheme == "https" { - tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) + tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host}) if err = tlsConn.Handshake(); err != nil { return From 12043e5a017ab2e9c4e2312c93f4466e9fbe8fb3 Mon Sep 17 00:00:00 2001 From: Yohan Legat Date: Thu, 1 Dec 2016 12:07:00 +0100 Subject: [PATCH 72/79] Resolve buger/gor#394 : Timeout for NO_CONTENT responses 1xx, 204 and 304 HTTP responses MUST NOT include a message body (see [RFC-2616](https://tools.ietf.org/html/rfc2616#section-4.4)). Also, a server MAY send a Content-Length header field in a 304 (Not Modified) and MUST NOT send a Content-Length header field in any response with a status code of 1xx (Informational) or 204 (No Content) (see [RFC-7230](https://tools.ietf.org/html/rfc7230#section-3.3.2)) --- http_client.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/http_client.go b/http_client.go index 8ee42e4..8007055 100644 --- a/http_client.go +++ b/http_client.go @@ -207,9 +207,14 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if bytes.Equal(proto.Header(c.respBuf, []byte("Transfer-Encoding")), []byte("chunked")) { chunked = true } else { - l := proto.Header(c.respBuf, []byte("Content-Length")) - if len(l) > 0 { - contentLength, _ = strconv.Atoi(string(l)) + status, _ := strconv.Atoi(string(proto.Status(c.respBuf))) + if (status >= 100 && status < 200) || status == 204 || status == 304 { + contentLength = 0 + } else { + l := proto.Header(c.respBuf, []byte("Content-Length")) + if len(l) > 0 { + contentLength, _ = strconv.Atoi(string(l)) + } } } From bea255ab113f631d3bf1d33add36dc7a0e2ef046 Mon Sep 17 00:00:00 2001 From: Alexandr Korsak Date: Thu, 1 Dec 2016 16:46:18 +0300 Subject: [PATCH 73/79] Register kafka output plugin in case of having passed settings --- plugins.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins.go b/plugins.go index a625f9b..6ee1516 100644 --- a/plugins.go +++ b/plugins.go @@ -143,5 +143,7 @@ func InitPlugins() { registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig) } - registerPlugin(NewKafkaOutput, "", &Settings.outputKafkaConfig) + if Settings.outputKafkaConfig.host != "" && Settings.outputKafkaConfig.topic != "" { + registerPlugin(NewKafkaOutput, "", &Settings.outputKafkaConfig) + } } From fa0e96b7efe0c345f230286ac09b2aaf558ee64e Mon Sep 17 00:00:00 2001 From: Yohan Legat Date: Tue, 29 Nov 2016 14:07:41 +0100 Subject: [PATCH 74/79] Resolve buger/gor#392 : use pcap timestamp Timestamp requests are now fetched from pcap payload --- raw_socket_listener/listener.go | 54 +++++++++++------- raw_socket_listener/listener_test.go | 76 ++++++++++++------------- raw_socket_listener/tcp_message.go | 9 ++- raw_socket_listener/tcp_message_test.go | 57 ++++++++++++------- raw_socket_listener/tcp_packet.go | 35 +++++++----- 5 files changed, 135 insertions(+), 96 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index cecf979..1b1c8c0 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -33,6 +33,12 @@ import ( var _ = fmt.Println +type Packet struct { + srcIP []byte + data []byte + timestamp time.Time +} + // Listener handle traffic capture type Listener struct { mu sync.Mutex @@ -53,7 +59,7 @@ type Listener struct { respWithoutReq map[uint32]tcpID // Messages ready to be send to client - packetsChan chan []byte + packetsChan chan *Packet // Messages ready to be send to client messagesChan chan *TCPMessage @@ -88,7 +94,7 @@ const ( func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration) (l *Listener) { l = &Listener{} - l.packetsChan = make(chan []byte, 10000) + l.packetsChan = make(chan *Packet, 10000) l.messagesChan = make(chan *TCPMessage, 10000) l.quit = make(chan bool) l.readyCh = make(chan bool, 1) @@ -137,9 +143,9 @@ func (t *Listener) listen() { t.conn.Close() } return - case data := <-t.packetsChan: - packet := ParseTCPPacket(data[:16], data[16:]) - t.processTCPPacket(packet) + case packet := <-t.packetsChan: + tcpPacket := ParseTCPPacket(packet.srcIP, packet.data, packet.timestamp) + t.processTCPPacket(tcpPacket) case <-gcTicker: now := time.Now() @@ -522,11 +528,13 @@ func (t *Listener) readPcap() { } } - newBuf := make([]byte, len(data)+16) - copy(newBuf[:16], srcIP) - copy(newBuf[16:], data) + packetSrcIP := make([]byte, 16) + packetData := make([]byte, len(data)) - t.packetsChan <- newBuf + copy(packetSrcIP, srcIP) + copy(packetData, data) + + t.packetsChan <- t.buildPacket(srcIP, data, packet.Metadata().Timestamp) } } }(d) @@ -589,11 +597,7 @@ func (t *Listener) readPcapFile() { continue } - newBuf := make([]byte, len(data)+16) - copy(newBuf[:16], addr) - copy(newBuf[16:], data) - - t.packetsChan <- newBuf + t.packetsChan <- t.buildPacket(addr, data, packet.Metadata().Timestamp) } } } @@ -626,16 +630,26 @@ func (t *Listener) readRAWSocket() { if n > 0 { if t.isValidPacket(buf[:n]) { - newBuf := make([]byte, n+16) - copy(newBuf[16:], buf[:n]) - copy(newBuf[:16], []byte(addr.(*net.IPAddr).IP)) - - t.packetsChan <- newBuf + t.packetsChan <- t.buildPacket([]byte(addr.(*net.IPAddr).IP), buf[:n], time.Now()) } } } } +func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *Packet { + copyPacketSrcIP := make([]byte, 16) + copyPacketData := make([]byte, len(packetData)) + + copy(copyPacketSrcIP, packetSrcIP) + copy(copyPacketData, packetSrcIP) + + return &Packet { + srcIP: packetSrcIP, + data: packetData, + timestamp:timestamp, + } +} + func (t *Listener) isValidPacket(buf []byte) bool { // To avoid full packet parsing every time, we manually parsing values needed for packet filtering // http://en.wikipedia.org/wiki/Transmission_Control_Protocol @@ -718,7 +732,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { message, ok := t.messages[packet.ID] if !ok { - message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming) + message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, packet.timestamp) t.messages[packet.ID] = message if !isIncoming { diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index 72e01de..8ade26e 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -15,10 +15,10 @@ func TestRawListenerInput(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) respAck := reqPacket.Seq + uint32(len(reqPacket.Data)) - respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) listener.packetsChan <- reqPacket.Dump() listener.packetsChan <- respPacket.Dump() @@ -52,11 +52,11 @@ func TestRawListenerInputResponseByClose(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) respAck := reqPacket.Seq + uint32(len(reqPacket.Data)) - respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd")) - finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte("")) + respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd"), time.Now()) + finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""), time.Now()) finPacket.IsFIN = true listener.packetsChan <- reqPacket.Dump() @@ -92,7 +92,7 @@ func TestRawListenerInputWithoutResponse(t *testing.T) { listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) listener.packetsChan <- reqPacket.Dump() @@ -114,8 +114,8 @@ func TestRawListenerResponse(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) - respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) + respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) // If response packet comes before request listener.packetsChan <- respPacket.Dump() @@ -152,15 +152,15 @@ func TestShort100Continue(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n")) + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now()) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") @@ -172,15 +172,15 @@ func Test100ContinueWrongOrder(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n")) + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now()) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") @@ -191,15 +191,15 @@ func TestAlt100ContinueHeaderOrder(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n")) + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now()) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") @@ -352,16 +352,16 @@ func TestRawListenerChunkedWrongOrder(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n")) + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n")) - reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n")) + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n"), time.Now()) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n"), time.Now()) + reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n"), time.Now()) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now()) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) // Should re-construct message from all possible combinations for i := 0; i < 6*5*4*3*2*1; i++ { @@ -381,13 +381,13 @@ func chunkedPostMessage() []*TCPPacket { ack := uint32(rand.Int63()) seq := uint32(rand.Int63()) - reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n")) + reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n")) - reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n")) - reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n")) + reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"), time.Now()) + reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"), time.Now()) + reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"), time.Now()) - respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) return []*TCPPacket{ reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket, @@ -409,8 +409,8 @@ func postMessage() []*TCPPacket { } return []*TCPPacket{ - buildPacket(true, ack, seq, data), - buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n")), + buildPacket(true, ack, seq, data, time.Now()), + buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()), } } @@ -420,8 +420,8 @@ func getMessage() []*TCPPacket { seq := uint32(rand.Int63()) return []*TCPPacket{ - buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n")), - buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n")), + buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()), + buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()), } } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 5cb566e..5319594 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -48,9 +48,8 @@ type TCPMessage struct { } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(Seq, Ack uint32, IsIncoming bool) (msg *TCPMessage) { - msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming} - msg.Start = time.Now() +func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, timestamp time.Time) (msg *TCPMessage) { + msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, Start: timestamp} return } @@ -138,6 +137,10 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { if packet.OrigAck != 0 { t.DataAck = packet.OrigAck } + + if packet.timestamp.Before(t.Start) { + t.Start = packet.timestamp + } } t.checkSeqIntegrity() diff --git a/raw_socket_listener/tcp_message_test.go b/raw_socket_listener/tcp_message_test.go index 781a824..1ce0d19 100644 --- a/raw_socket_listener/tcp_message_test.go +++ b/raw_socket_listener/tcp_message_test.go @@ -5,9 +5,10 @@ import ( "encoding/binary" _ "log" "testing" + "time" ) -func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte) (packet *TCPPacket) { +func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte, timestamp time.Time) (packet *TCPPacket) { var srcPort, destPort uint16 // For tests `listening` port is 0 @@ -25,7 +26,7 @@ func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte) (packet *TCPPack buf[12] = 64 buf = append(buf, Data...) - packet = ParseTCPPacket([]byte("123"), buf) + packet = ParseTCPPacket([]byte("123"), buf, timestamp) return packet } @@ -36,31 +37,31 @@ func buildMessage(p *TCPPacket) *TCPMessage { isIncoming = true } - m := NewTCPMessage(p.Seq, p.Ack, isIncoming) + m := NewTCPMessage(p.Seq, p.Ack, isIncoming, p.timestamp) m.AddPacket(p) return m } func TestTCPMessagePacketsOrder(t *testing.T) { - msg := buildMessage(buildPacket(true, 1, 1, []byte("a"))) - msg.AddPacket(buildPacket(true, 1, 2, []byte("b"))) + msg := buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now())) if !bytes.Equal(msg.Bytes(), []byte("ab")) { t.Error("Should contatenate packets in right order") } // When first packet have wrong order (Seq) - msg = buildMessage(buildPacket(true, 1, 2, []byte("b"))) - msg.AddPacket(buildPacket(true, 1, 1, []byte("a"))) + msg = buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now())) if !bytes.Equal(msg.Bytes(), []byte("ab")) { t.Error("Should contatenate packets in right order") } // Should ignore packets with same sequence - msg = buildMessage(buildPacket(true, 1, 1, []byte("a"))) - msg.AddPacket(buildPacket(true, 1, 1, []byte("a"))) + msg = buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now())) if !bytes.Equal(msg.Bytes(), []byte("a")) { t.Error("Should ignore packet with same Seq") @@ -68,8 +69,8 @@ func TestTCPMessagePacketsOrder(t *testing.T) { } func TestTCPMessageSize(t *testing.T) { - msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"))) - msg.AddPacket(buildPacket(true, 1, 2, []byte("b"))) + msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now())) if msg.BodySize() != 2 { t.Error("Should count only body", msg.BodySize()) @@ -110,7 +111,7 @@ func TestTCPMessageIsComplete(t *testing.T) { } for _, tc := range testCases { - msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now())) if tc.assocMessage { msg.AssocMessage = &TCPMessage{} } @@ -123,9 +124,9 @@ func TestTCPMessageIsComplete(t *testing.T) { } func TestTCPMessageIsSeqMissing(t *testing.T) { - p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n")) - p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n")) - p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a")) + p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"), time.Now()) + p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now()) + p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"), time.Now()) msg := buildMessage(p1) if msg.seqMissing { @@ -144,8 +145,8 @@ func TestTCPMessageIsSeqMissing(t *testing.T) { } func TestTCPMessageIsHeadersReceived(t *testing.T) { - p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n")) - p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n")) + p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) + p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now()) msg := buildMessage(p1) if msg.headerPacket == -1 { @@ -157,7 +158,7 @@ func TestTCPMessageIsHeadersReceived(t *testing.T) { t.Error("Should found double new line: headers received") } - msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"))) + msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"), time.Now())) if msg.headerPacket != -1 { t.Error("Should not find headers end") } @@ -183,7 +184,7 @@ func TestTCPMessageMethodType(t *testing.T) { } for _, tc := range testCases { - msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now())) if msg.methodType != tc.expectedMethodType { t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType) @@ -208,7 +209,7 @@ func TestTCPMessageBodyType(t *testing.T) { } for _, tc := range testCases { - msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now())) if msg.bodyType != tc.expectedBodyType { t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType) @@ -229,12 +230,12 @@ func TestTCPMessageBodySize(t *testing.T) { } for _, tc := range testCases { - msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]))) + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]), time.Now())) if len(tc.payloads) > 1 { for _, p := range tc.payloads[1:] { seq := uint32(1 + msg.Size()) - msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p))) + msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p), time.Now())) } } @@ -243,3 +244,15 @@ func TestTCPMessageBodySize(t *testing.T) { } } } + +func TestTcpMessageStart(t *testing.T) { + start := time.Now().Add(-1 * time.Second) + + msg := buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), start)) + + if msg.Start != start { + t.Error("Message timestamp should be equal to the lowest related packet timestamp", start, msg.Start) + } +} + diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 0e3e832..1dafcbe 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -5,6 +5,7 @@ import ( "log" "strconv" "strings" + "time" ) var _ = log.Println @@ -38,14 +39,16 @@ type TCPPacket struct { Raw []byte Data []byte Addr []byte + timestamp time.Time ID tcpID } // ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket -func ParseTCPPacket(addr []byte, data []byte) (p *TCPPacket) { +func ParseTCPPacket(addr []byte, data []byte, timestamp time.Time) (p *TCPPacket) { p = &TCPPacket{Raw: data} p.ParseBasic() p.Addr = addr + p.timestamp = timestamp p.GenID() return @@ -79,27 +82,33 @@ func (t *TCPPacket) ParseBasic() { t.Data = t.Raw[t.DataOffset*4:] } -func (t *TCPPacket) Dump() []byte { - buf := make([]byte, len(t.Data)+16+16) - copy(buf[:16], t.Addr) +func (t *TCPPacket) Dump() *Packet { - tcpBuf := buf[16:] + packetSrcIP := make([]byte, 16) + packetData := make([]byte, len(t.Data) + 16) - binary.BigEndian.PutUint16(tcpBuf[2:4], t.DestPort) - binary.BigEndian.PutUint16(tcpBuf[0:2], t.SrcPort) + copy(packetSrcIP, t.Addr) - binary.BigEndian.PutUint32(tcpBuf[4:8], t.Seq) - binary.BigEndian.PutUint32(tcpBuf[8:12], t.Ack) + binary.BigEndian.PutUint16(packetData[0:2], t.SrcPort) + binary.BigEndian.PutUint16(packetData[2:4], t.DestPort) - tcpBuf[12] = 64 + binary.BigEndian.PutUint32(packetData[4:8], t.Seq) + binary.BigEndian.PutUint32(packetData[8:12], t.Ack) + + packetData[12] = 64 if t.IsFIN { - tcpBuf[13] = tcpBuf[13] | 0x01 + packetData[13] = packetData[13] | 0x01 } - copy(tcpBuf[16:], t.Data) + copy(packetData[16:], t.Data) + + return &Packet{ + srcIP: packetSrcIP, + data:packetData, + timestamp:t.timestamp, + } - return buf } // String output for a TCP Packet From c6244435caa8f0658905c7fac73564a5e4b53136 Mon Sep 17 00:00:00 2001 From: Yohan Legat Date: Thu, 1 Dec 2016 10:52:21 +0100 Subject: [PATCH 75/79] [Boyscout] unexport TCPPacket.Dump() and Packet struct --- raw_socket_listener/listener.go | 10 +++++----- raw_socket_listener/listener_test.go | 22 +++++++++++----------- raw_socket_listener/tcp_packet.go | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 1b1c8c0..8428ba7 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -33,7 +33,7 @@ import ( var _ = fmt.Println -type Packet struct { +type packet struct { srcIP []byte data []byte timestamp time.Time @@ -59,7 +59,7 @@ type Listener struct { respWithoutReq map[uint32]tcpID // Messages ready to be send to client - packetsChan chan *Packet + packetsChan chan *packet // Messages ready to be send to client messagesChan chan *TCPMessage @@ -94,7 +94,7 @@ const ( func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration) (l *Listener) { l = &Listener{} - l.packetsChan = make(chan *Packet, 10000) + l.packetsChan = make(chan *packet, 10000) l.messagesChan = make(chan *TCPMessage, 10000) l.quit = make(chan bool) l.readyCh = make(chan bool, 1) @@ -636,14 +636,14 @@ func (t *Listener) readRAWSocket() { } } -func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *Packet { +func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet { copyPacketSrcIP := make([]byte, 16) copyPacketData := make([]byte, len(packetData)) copy(copyPacketSrcIP, packetSrcIP) copy(copyPacketData, packetSrcIP) - return &Packet { + return &packet{ srcIP: packetSrcIP, data: packetData, timestamp:timestamp, diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index 8ade26e..70271a7 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -20,8 +20,8 @@ func TestRawListenerInput(t *testing.T) { respAck := reqPacket.Seq + uint32(len(reqPacket.Data)) respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) - listener.packetsChan <- reqPacket.Dump() - listener.packetsChan <- respPacket.Dump() + listener.packetsChan <- reqPacket.dump() + listener.packetsChan <- respPacket.dump() select { case req = <-listener.messagesChan: @@ -59,9 +59,9 @@ func TestRawListenerInputResponseByClose(t *testing.T) { finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""), time.Now()) finPacket.IsFIN = true - listener.packetsChan <- reqPacket.Dump() - listener.packetsChan <- respPacket.Dump() - listener.packetsChan <- finPacket.Dump() + listener.packetsChan <- reqPacket.dump() + listener.packetsChan <- respPacket.dump() + listener.packetsChan <- finPacket.dump() select { case req = <-listener.messagesChan: @@ -94,7 +94,7 @@ func TestRawListenerInputWithoutResponse(t *testing.T) { reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) - listener.packetsChan <- reqPacket.Dump() + listener.packetsChan <- reqPacket.dump() select { case req = <-listener.messagesChan: @@ -118,8 +118,8 @@ func TestRawListenerResponse(t *testing.T) { respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) // If response packet comes before request - listener.packetsChan <- respPacket.Dump() - listener.packetsChan <- reqPacket.Dump() + listener.packetsChan <- respPacket.dump() + listener.packetsChan <- reqPacket.dump() select { case req = <-listener.messagesChan: @@ -209,7 +209,7 @@ func TestAlt100ContinueHeaderOrder(t *testing.T) { func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) { var req, resp *TCPMessage for _, p := range packets { - listener.packetsChan <- p.Dump() + listener.packetsChan <- p.dump() } select { @@ -249,7 +249,7 @@ func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket var r, req, resp *TCPMessage for _, p := range packets { - listener.packetsChan <- p.Dump() + listener.packetsChan <- p.dump() } select { @@ -452,7 +452,7 @@ func TestRawListenerBench(t *testing.T) { } } - l.packetsChan <- p.Dump() + l.packetsChan <- p.dump() time.Sleep(time.Millisecond) } diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 1dafcbe..f2ba82c 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -82,7 +82,7 @@ func (t *TCPPacket) ParseBasic() { t.Data = t.Raw[t.DataOffset*4:] } -func (t *TCPPacket) Dump() *Packet { +func (t *TCPPacket) dump() *packet { packetSrcIP := make([]byte, 16) packetData := make([]byte, len(t.Data) + 16) @@ -103,7 +103,7 @@ func (t *TCPPacket) Dump() *Packet { copy(packetData[16:], t.Data) - return &Packet{ + return &packet{ srcIP: packetSrcIP, data:packetData, timestamp:t.timestamp, From b462c06a7c2f28ec08e88f504992563c00e80c89 Mon Sep 17 00:00:00 2001 From: Yohan Legat Date: Thu, 8 Dec 2016 17:35:45 +0100 Subject: [PATCH 76/79] remove dead code --- raw_socket_listener/listener.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 8428ba7..9a8b83f 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -528,12 +528,6 @@ func (t *Listener) readPcap() { } } - packetSrcIP := make([]byte, 16) - packetData := make([]byte, len(data)) - - copy(packetSrcIP, srcIP) - copy(packetData, data) - t.packetsChan <- t.buildPacket(srcIP, data, packet.Metadata().Timestamp) } } From 0a74db6aae5c0c4dba0af0622637a6a87c9a51f3 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 18 Dec 2016 20:09:24 +0300 Subject: [PATCH 77/79] Disallow zero timeout (also used fix tests) --- http_client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/http_client.go b/http_client.go index 8007055..aeb322f 100644 --- a/http_client.go +++ b/http_client.go @@ -59,6 +59,10 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { u, _ := url.Parse(baseURL) + if config.Timeout == 0 { + config.Timeout = time.Second + } + config.ConnectionTimeout = config.Timeout if config.ResponseBufferSize == 0 { From 708b2cebd36a86e38b5fd922dba902ff181baddc Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 19 Dec 2016 14:17:13 +0300 Subject: [PATCH 78/79] change --- s3/index.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/s3/index.html b/s3/index.html index 3502792..c4ac524 100644 --- a/s3/index.html +++ b/s3/index.html @@ -38,6 +38,12 @@ ul {

    Gor PRO releases

    See releases page on GitHub for changelog

    +

    v0.15.1

    + +

    v0.15.0

    • gor_v0.15.0_PRO_x64.tar.gz - Linux x64
      • From c4869d93386a48b473f54c4ff0825b039a968917 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 19 Dec 2016 14:49:28 +0300 Subject: [PATCH 79/79] fix --- http_client_test.go | 4 ++-- raw_socket_listener/listener.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/http_client_test.go b/http_client_test.go index 00f9db3..917c7c9 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -24,8 +24,8 @@ func TestHTTPClientURLPort(t *testing.T) { } c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{}) - if c2.baseURL != "https://example.com:443" { - t.Error("Sould add 443 port for https:", c2.baseURL) + if c2.baseURL != "https://example.com" { + t.Error("Sould not add 443 port for https:", c2.baseURL) } c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{}) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index cf7e38a..a851fb5 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -731,7 +731,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { message, ok := t.messages[packet.ID] if !ok { - message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol, packet.timestamp)) + message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol, packet.timestamp) t.messages[packet.ID] = message if !isIncoming {