diff --git a/output_http.go b/output_http.go index b55002c..937c815 100644 --- a/output_http.go +++ b/output_http.go @@ -2,6 +2,7 @@ package main import ( "io" + "log" "sync/atomic" "time" @@ -21,7 +22,9 @@ type response struct { type HTTPOutputConfig struct { redirectLimit int - stats bool + stats bool + workersMin int + workersMax int statsMs int workers int queueLen int @@ -78,10 +81,10 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o.needWorker = make(chan int, 1) // Initial workers count - if o.config.workers == 0 { + if o.config.workersMax == 0 { o.needWorker <- initialDynamicWorkers } else { - o.needWorker <- o.config.workers + o.needWorker <- o.config.workersMax } if o.config.elasticSearch != "" { @@ -100,11 +103,6 @@ func (o *HTTPOutput) workerMaster() { for i := 0; i < newWorkers; i++ { go o.startWorker() } - - // Disable dynamic scaling if workers poll fixed size - if o.config.workers != 0 { - return - } } } @@ -128,17 +126,16 @@ func (o *HTTPOutput) startWorker() { deathCount = 0 case <-time.After(time.Millisecond * 100): // When dynamic scaling enabled workers die after 2s of inactivity - if o.config.workers == 0 { - deathCount++ - } else { + if o.config.workersMin == o.config.workersMax { continue } + deathCount++ if deathCount > 20 { - workersCount := atomic.LoadInt64(&o.activeWorkers) + workersCount := int(atomic.LoadInt64(&o.activeWorkers)) // At least 1 startWorker should be alive - if workersCount != 1 { + if workersCount != 1 && workersCount > o.config.workersMin { atomic.AddInt64(&o.activeWorkers, -1) return } @@ -161,11 +158,18 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { o.queueStats.Write(len(o.queue)) } - if o.config.workers == 0 { - workersCount := atomic.LoadInt64(&o.activeWorkers) + if o.config.workersMax != o.config.workersMin { + workersCount := int(atomic.LoadInt64(&o.activeWorkers)) - if len(o.queue) > int(workersCount) { - o.needWorker <- len(o.queue) + if len(o.queue) > workersCount { + extraWorkersReq := len(o.queue) - workersCount + 1 + maxWorkersAvailable := o.config.workersMax - workersCount + if extraWorkersReq > maxWorkersAvailable { + extraWorkersReq = maxWorkersAvailable + } + if extraWorkersReq > 0 { + o.needWorker <- extraWorkersReq + } } } @@ -208,6 +212,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { stop := time.Now() if err != nil { + log.Println("Error when sending ", err, time.Now()) Debug("Request error:", err) } diff --git a/settings.go b/settings.go index ac18857..1fdf628 100644 --- a/settings.go +++ b/settings.go @@ -149,8 +149,11 @@ func init() { flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") - flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") + + flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.") + flag.IntVar(&Settings.outputHTTPConfig.workersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.") flag.IntVar(&Settings.outputHTTPConfig.queueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000") + flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") flag.BoolVar(&Settings.outputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.")