Track http responses (#416)

emitter.go

@@ -27,6 +27,12 @@ func Start(stop chan int) {
 		for _, in := range Plugins.Inputs {
 			go CopyMulty(in, Plugins.Outputs...)
 		}
+
+		for _, out := range Plugins.Outputs {
+			if r, ok := out.(io.Reader); ok {
+				go CopyMulty(r, Plugins.Outputs...)
+			}
+		}
 	}
 
 	for {

output_file.go

@@ -186,10 +186,6 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 		o.updateName()
 	}
 
-	if !isOriginPayload(data) {
-		return len(data), nil
-	}
-
 	if o.file == nil || o.currentName != o.file.Name() {
 		o.mu.Lock()
 		o.Close()

output_http.go

@@ -87,10 +87,6 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 		o.elasticSearch.Init(o.config.elasticSearch)
 	}
 
-	if len(Settings.middleware) > 0 {
-		o.config.TrackResponses = true
-	}
-
 	go o.workerMaster()
 
 	return o

output_http_test.go

@@ -43,15 +43,19 @@ func TestHTTPOutput(t *testing.T) {
 	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
 	Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods}
 
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
+	http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true, TrackResponses: true})
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
 
 	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	Plugins.Outputs = []io.Writer{http_output, output}
 
 	go Start(quit)
 
-	for i := 0; i < 100; i++ {
-		wg.Add(2) // OPTIONS should be ignored
+	for i := 0; i < 1; i++ {
+		// 2 http-output, 2 - test output request, 2 - test output http response
+		wg.Add(6) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()

settings.go

@@ -122,6 +122,7 @@ func init() {
 	flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
 	flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
 	flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
+	flag.BoolVar(&Settings.outputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.")
 
 	flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
 	flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")