Merge remote-tracking branch 'upstream/master'

emitter.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"hash/fnv"
 	"io"
+	"log"
 	"time"
 )
 
@@ -23,15 +24,30 @@ func Start(stop chan int) {
 			}
 		}
 
-		go CopyMulty(middleware, Plugins.Outputs...)
+		go func() {
+			if err := CopyMulty(middleware, Plugins.Outputs...); err != nil {
+				log.Println("Error during copy: ", err)
+				close(stop)
+			}
+		}()
 	} else {
 		for _, in := range Plugins.Inputs {
-			go CopyMulty(in, Plugins.Outputs...)
+			go func() {
+				if err := CopyMulty(in, Plugins.Outputs...); err != nil {
+					log.Println("Error during copy: ", err)
+					close(stop)
+				}
+			}()
 		}
 
 		for _, out := range Plugins.Outputs {
 			if r, ok := out.(io.Reader); ok {
-				go CopyMulty(r, Plugins.Outputs...)
+				go func() {
+					if err := CopyMulty(r, Plugins.Outputs...); err != nil {
+						log.Println("Error during copy: ", err)
+						close(stop)
+					}
+				}()
 			}
 		}
 	}
@@ -48,7 +64,7 @@ func Start(stop chan int) {
 
 // CopyMulty copies from 1 reader to multiple writers
 func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
-	buf := make([]byte, 5*1024*1024)
+	buf := make([]byte, Settings.copyBufferSize)
 	wIndex := 0
 	modifier := NewHTTPModifier(&Settings.modifierConfig)
 	filteredRequests := make(map[string]time.Time)
@@ -58,11 +74,18 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 
 	for {
 		nr, er := src.Read(buf)
+
+		if er == io.EOF {
+			return nil
+		}
+		if er != nil {
+			return err
+		}
+
 		_maxN := nr
 		if nr > 500 {
 			_maxN = 500
 		}
-
 		if nr > 0 && len(buf) > nr {
 			payload := buf[:nr]
 			meta := payloadMeta(payload)
@@ -74,6 +97,10 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 			}
 			requestID := string(meta[1])
 
+			if nr >= 5*1024*1024 {
+				log.Println("INFO: Large packet... We received ", len(payload), " bytes from ", src)
+			}
+
 			if Settings.debug {
 				Debug("[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
 			}
@@ -123,8 +150,10 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 					wIndex = int(hasher.Sum32()) % len(writers)
 					writers[wIndex].Write(payload)
 				} else {
-					// Simple round robin
-					writers[wIndex].Write(payload)
+				    // Simple round robin
+				    if _, err := writers[wIndex].Write(payload); err != nil {
+					    return err
+				    }
 
 					wIndex++
 
@@ -134,17 +163,13 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 				}
 			} else {
 				for _, dst := range writers {
-					dst.Write(payload)
+					if _, err := dst.Write(payload); err != nil {
+						return err
+					}
 				}
 			}
-
-		}
-		if er == io.EOF {
-			break
-		}
-		if er != nil {
-			err = er
-			break
+		} else if nr > 0 {
+			log.Println("WARN: Packet", nr, "bytes is too large to process. Consider increasing --copy-buffer-size")
 		}
 
 		// Run GC on each 1000 request