From 80b19f918bae7edeb1362085f44e186ca35edae0 Mon Sep 17 00:00:00 2001 From: arijitad Date: Wed, 15 Jul 2020 00:14:38 +0530 Subject: [PATCH] Map Appsetting to flag names. --- elasticsearch.go | 8 +- elasticsearch_test.go | 18 +- emitter.go | 14 +- emitter_test.go | 30 +- go.mod | 4 +- go.sum | 4 + gor.go | 16 +- gor_stat.go | 4 +- http_client_test.go | 2 +- input_file_test.go | 4 +- input_http_test.go | 4 +- input_kafka.go | 18 +- input_kafka_test.go | 12 +- input_raw.go | 2 +- input_raw_test.go | 18 +- input_tcp.go | 6 +- input_tcp_test.go | 4 +- kafka.go | 17 +- limiter_test.go | 8 +- middleware.go | 8 +- middleware_test.go | 12 +- output_binary.go | 10 +- output_file.go | 10 +- output_file_test.go | 4 +- output_http.go | 34 +-- output_http_test.go | 26 +- output_kafka.go | 12 +- output_kafka_test.go | 12 +- output_tcp.go | 8 +- output_tcp_test.go | 4 +- plugins.go | 56 ++-- plugins_test.go | 8 +- settings.go | 284 +++++++++--------- settings_test.go | 16 + .../aws/aws-sdk-go/aws/endpoints/defaults.go | 112 ++++++- .../github.com/aws/aws-sdk-go/aws/version.go | 2 +- vendor/modules.txt | 4 +- 37 files changed, 470 insertions(+), 345 deletions(-) create mode 100644 settings_test.go diff --git a/elasticsearch.go b/elasticsearch.go index 391b700..8e3ab27 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -15,7 +15,7 @@ import ( type ESUriErorr struct{} func (e *ESUriErorr) Error() string { - return "Wrong ElasticSearch URL format. Expected to be: scheme://host/index_name" + return "Wrong ElasticSearch URL format. Expected to be: scheme://Host/index_name" } type ESPlugin struct { @@ -55,7 +55,7 @@ type ESRequestResponse struct { // Parse ElasticSearch URI // -// Proper format is: scheme://[userinfo@]host/index_name +// Proper format is: scheme://[userinfo@]Host/index_name // userinfo is: user[:password] // net/url.Parse() does not fail if scheme is not provided but actualy does not // handle URI properly. @@ -69,7 +69,7 @@ func parseURI(URI string) (err error, index string) { return } - // check URL validity by extracting host and undex values. + // check URL validity by extracting Host and undex values. host := parsedUrl.Host urlPathParts := strings.Split(parsedUrl.Path, "/") index = urlPathParts[len(urlPathParts)-1] @@ -99,7 +99,7 @@ func (p *ESPlugin) Init(URI string) { p.done = make(chan bool) p.indexor.Start() - if Settings.verbose { + if Settings.Verbose { // Only start the ErrorHandler goroutine when in verbose mode // no need to burn ressources otherwise go p.ErrorHandler() diff --git a/elasticsearch_test.go b/elasticsearch_test.go index 7bddcff..6f810d4 100644 --- a/elasticsearch_test.go +++ b/elasticsearch_test.go @@ -32,7 +32,7 @@ func assertNoError(returnedError error, t *testing.T) { } } -// Argument host:port/index_name +// Argument Host:port/index_name // i.e : localhost:9200/gor // Fail because scheme is mandatory func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) { @@ -42,7 +42,7 @@ func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) { assertExpectedError(err, t) } -// Argument scheme://host:port +// Argument scheme://Host:port // i.e : http://localhost:9200 // Fail : explicit index is required func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) { @@ -55,7 +55,7 @@ func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) { assertExpectedError(err, t) } -// Argument scheme://host/index_name +// Argument scheme://Host/index_name // i.e : http://localhost/gor func TestElasticConnectionBuildFailWithoutPort(t *testing.T) { uri := "http://localhost/" + expectedIndex @@ -67,7 +67,7 @@ func TestElasticConnectionBuildFailWithoutPort(t *testing.T) { assertExpectedGorIndex(index, t) } -// Argument scheme://host:port/index_name +// Argument scheme://Host:port/index_name // i.e : http://localhost:9200/gor func TestElasticLocalConnectionBuild(t *testing.T) { uri := "http://localhost:9200/" + expectedIndex @@ -79,7 +79,7 @@ func TestElasticLocalConnectionBuild(t *testing.T) { assertExpectedGorIndex(index, t) } -// Argument scheme://host:port/index_name +// Argument scheme://Host:port/index_name // i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) { uri := "http://localhost.local:9200/" + expectedIndex @@ -91,7 +91,7 @@ func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) { assertExpectedGorIndex(index, t) } -// Argument scheme://host:port/index_name +// Argument scheme://Host:port/index_name // i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) { uri := "https://localhost.local:9200/" + expectedIndex @@ -103,7 +103,7 @@ func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) { assertExpectedGorIndex(index, t) } -// Argument scheme://host:port/index_name +// Argument scheme://Host:port/index_name // i.e : localhost.local:9200/pathtoElastic/gor func TestElasticLongPathConnectionBuild(t *testing.T) { uri := "http://localhost.local:9200/pathtoElastic/" + expectedIndex @@ -115,7 +115,7 @@ func TestElasticLongPathConnectionBuild(t *testing.T) { assertExpectedGorIndex(index, t) } -// Argument scheme://host:userinfo@port/index_name +// Argument scheme://Host:userinfo@port/index_name // i.e : http://user:password@localhost.local:9200/gor func TestElasticBasicAuthConnectionBuild(t *testing.T) { uri := "http://user:password@localhost.local:9200/" + expectedIndex @@ -127,7 +127,7 @@ func TestElasticBasicAuthConnectionBuild(t *testing.T) { assertExpectedGorIndex(index, t) } -// Argument scheme://host:port/path/index_name +// Argument scheme://Host:port/path/index_name // i.e : http://localhost.local:9200/path/gor or https://localhost.local:9200/path/gor func TestElasticComplexPathConnectionBuild(t *testing.T) { uri := "http://localhost.local:9200/path/" + expectedIndex diff --git a/emitter.go b/emitter.go index 396b7b0..02d3e31 100644 --- a/emitter.go +++ b/emitter.go @@ -110,7 +110,7 @@ func (e *emitter) Close() { func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error { buf := make([]byte, Settings.copyBufferSize) wIndex := 0 - modifier := NewHTTPModifier(&Settings.modifierConfig) + modifier := NewHTTPModifier(&Settings.ModifierConfig) filteredRequests := make(map[string]time.Time) filteredRequestsLastCleanTime := time.Now() @@ -140,7 +140,7 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error { payload := buf[:nr] meta := payloadMeta(payload) if len(meta) < 3 { - if Settings.debug { + if Settings.Debug { Debug("[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src) } continue @@ -151,7 +151,7 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error { log.Println("INFO: Large packet... We received ", len(payload), " bytes from ", src) } - if Settings.debug { + if Settings.Debug { Debug("[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src) } @@ -172,7 +172,7 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error { payload = append(payload[:headSize], body...) } - if Settings.debug { + if Settings.Debug { Debug("[EMITTER] Rewritten input:", len(payload), "First 500 bytes:", string(payload[0:_maxN])) } } else { @@ -183,15 +183,15 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error { } } - if Settings.prettifyHTTP { + if Settings.PrettifyHTTP { payload = prettifyHTTP(payload) if len(payload) == 0 { continue } } - if Settings.splitOutput { - if Settings.recognizeTCPSessions { + if Settings.SplitOutput { + if Settings.RecognizeTCPSessions { if !PRO { log.Fatal("Detailed TCP sessions work only with PRO license") } diff --git a/emitter_test.go b/emitter_test.go index b34fb3c..0550f90 100644 --- a/emitter_test.go +++ b/emitter_test.go @@ -32,7 +32,7 @@ func TestEmitter(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 1000; i++ { wg.Add(1) @@ -61,7 +61,7 @@ func TestEmitterFiltered(t *testing.T) { plugins.All = append(plugins.All, input, output) methods := HTTPMethods{[]byte("GET")} - Settings.modifierConfig = HTTPModifierConfig{methods: methods} + Settings.ModifierConfig = HTTPModifierConfig{methods: methods} emitter := &emitter{quit: quit} go emitter.Start(plugins, "") @@ -91,7 +91,7 @@ func TestEmitterFiltered(t *testing.T) { wg.Wait() emitter.Close() - Settings.modifierConfig = HTTPModifierConfig{} + Settings.ModifierConfig = HTTPModifierConfig{} } func TestEmitterSplitRoundRobin(t *testing.T) { @@ -117,10 +117,10 @@ func TestEmitterSplitRoundRobin(t *testing.T) { Outputs: []io.Writer{output1, output2}, } - Settings.splitOutput = true + Settings.SplitOutput = true emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 1000; i++ { wg.Add(1) @@ -135,7 +135,7 @@ func TestEmitterSplitRoundRobin(t *testing.T) { t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2) } - Settings.splitOutput = false + Settings.SplitOutput = false } func TestEmitterRoundRobin(t *testing.T) { @@ -162,10 +162,10 @@ func TestEmitterRoundRobin(t *testing.T) { } plugins.All = append(plugins.All, input, output1, output2) - Settings.splitOutput = true + Settings.SplitOutput = true emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 1000; i++ { wg.Add(1) @@ -179,7 +179,7 @@ func TestEmitterRoundRobin(t *testing.T) { t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2) } - Settings.splitOutput = false + Settings.SplitOutput = false } func TestEmitterSplitSession(t *testing.T) { @@ -222,11 +222,11 @@ func TestEmitterSplitSession(t *testing.T) { Outputs: []io.Writer{output1, output2}, } - Settings.splitOutput = true - Settings.recognizeTCPSessions = true + Settings.SplitOutput = true + Settings.RecognizeTCPSessions = true emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 1000; i++ { // Keep session but randomize ACK @@ -247,8 +247,8 @@ func TestEmitterSplitSession(t *testing.T) { t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2) } - Settings.splitOutput = false - Settings.recognizeTCPSessions = false + Settings.SplitOutput = false + Settings.RecognizeTCPSessions = false emitter.Close() } @@ -269,7 +269,7 @@ func BenchmarkEmitter(b *testing.B) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) b.ResetTimer() diff --git a/go.mod b/go.mod index 78b9a42..1eea393 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/Shopify/sarama v1.26.4 github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect - github.com/aws/aws-sdk-go v1.32.7 + github.com/aws/aws-sdk-go v1.33.2 github.com/bitly/go-hostpool v0.1.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/google/gopacket v1.1.17 @@ -15,5 +15,5 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect - golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect + golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect ) diff --git a/go.sum b/go.sum index e0b1764..aeea185 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 h1:Xz25cuW4REGC5W5UtpM github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8= github.com/aws/aws-sdk-go v1.32.7 h1:H4VgdCSF1cHw0VD8zGc98T1bGdACoLkh/vK2L6wgOUU= github.com/aws/aws-sdk-go v1.32.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.33.2 h1:8TVrnPnSD7I+AmDp66xBUvS3K0J+jH09YXdrkJ34ey0= +github.com/aws/aws-sdk-go v1.33.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= @@ -84,6 +86,8 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjut golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/gor.go b/gor.go index d7caca7..2720cd3 100644 --- a/gor.go +++ b/gor.go @@ -41,7 +41,7 @@ func main() { // defer func() { // if r := recover(); r != nil { // fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack()) - // } + // }̦ // }() // If not set via env cariable @@ -80,9 +80,9 @@ func main() { profileCPU(*cpuprofile) } - if Settings.pprof != "" { + if Settings.Pprof != "" { go func() { - log.Println(http.ListenAndServe(Settings.pprof, nil)) + log.Println(http.ListenAndServe(Settings.Pprof, nil)) }() } @@ -95,16 +95,16 @@ func main() { os.Exit(1) }() - if Settings.exitAfter > 0 { - log.Println("Running gor for a duration of", Settings.exitAfter) + if Settings.ExitAfter > 0 { + log.Println("Running gor for a duration of", Settings.ExitAfter) - time.AfterFunc(Settings.exitAfter, func() { - log.Println("Stopping gor after", Settings.exitAfter) + time.AfterFunc(Settings.ExitAfter, func() { + log.Println("Stopping gor after", Settings.ExitAfter) close(closeCh) }) } - emitter.Start(plugins, Settings.middleware) + emitter.Start(plugins, Settings.Middleware) } func finalize(plugins *InOutPlugins) { diff --git a/gor_stat.go b/gor_stat.go index 8f28e6f..a761f19 100644 --- a/gor_stat.go +++ b/gor_stat.go @@ -25,7 +25,7 @@ func NewGorStat(statName string, rateMs int) (s *GorStat) { s.max = 0 s.count = 0 - if Settings.stats { + if Settings.Stats { log.Println(s.statName + ":latest,mean,max,count,count/second,gcount") go s.reportStats() } @@ -33,7 +33,7 @@ func NewGorStat(statName string, rateMs int) (s *GorStat) { } func (s *GorStat) Write(latest int) { - if Settings.stats { + if Settings.Stats { if latest > s.max { s.max = latest } diff --git a/http_client_test.go b/http_client_test.go index f29701f..313b6e9 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -484,7 +484,7 @@ func TestHTTPClientErrors(t *testing.T) { client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { - t.Error("Should return status 521 for no such host, instead:", string(s)) + t.Error("Should return status 521 for no such Host, instead:", string(s)) } } else { t.Error("Should throw error") diff --git a/input_file_test.go b/input_file_test.go index 89cc572..f196fb4 100644 --- a/input_file_test.go +++ b/input_file_test.go @@ -329,7 +329,7 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile { plugins.All = append(plugins.All, output, outputFile) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) requestGenerator.emit() requestGenerator.wg.Wait() @@ -359,7 +359,7 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback wg.Add(count) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) done := make(chan int, 1) go func() { diff --git a/input_http_test.go b/input_http_test.go index a4f8e14..f0cad6c 100644 --- a/input_http_test.go +++ b/input_http_test.go @@ -28,7 +28,7 @@ func TestHTTPInput(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1) @@ -65,7 +65,7 @@ func TestInputHTTPLargePayload(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) wg.Add(1) address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1) diff --git a/input_kafka.go b/input_kafka.go index 90c17cd..2194439 100644 --- a/input_kafka.go +++ b/input_kafka.go @@ -12,13 +12,13 @@ import ( // KafkaInput is used for recieving Kafka messages and // transforming them into HTTP payloads. type KafkaInput struct { - config *KafkaConfig + config *InputKafkaConfig consumers []sarama.PartitionConsumer messages chan *sarama.ConsumerMessage } // NewKafkaInput creates instance of kafka consumer client. -func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput { +func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput { c := sarama.NewConfig() // Configuration options go here @@ -28,15 +28,15 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput { con = config.consumer } else { var err error - //con, err = sarama.NewConsumer([]string{config.host}, c) - con, err = sarama.NewConsumer(strings.Split(config.host, ","), c) + //con, err = sarama.NewConsumer([]string{config.Host}, c) + con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c) if err != nil { log.Fatalln("Failed to start Sarama(Kafka) consumer:", err) } } - partitions, err := con.Partitions(config.topic) + partitions, err := con.Partitions(config.Topic) if err != nil { log.Fatalln("Failed to collect Sarama(Kafka) partitions:", err) } @@ -48,7 +48,7 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput { } for index, partition := range partitions { - consumer, err := con.ConsumePartition(config.topic, partition, sarama.OffsetNewest) + consumer, err := con.ConsumePartition(config.Topic, partition, sarama.OffsetNewest) if err != nil { log.Fatalln("Failed to start Sarama(Kafka) partition consumer:", err) } @@ -61,7 +61,7 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput { } }(consumer) - if Settings.verbose { + if Settings.Verbose { // Start infinite loop for tracking errors for kafka producer. go i.ErrorHandler(consumer) } @@ -82,7 +82,7 @@ func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) { func (i *KafkaInput) Read(data []byte) (int, error) { message := <-i.messages - if !i.config.useJSON { + if !i.config.UseJSON { copy(data, message.Value) return len(message.Value), nil } @@ -103,5 +103,5 @@ func (i *KafkaInput) Read(data []byte) (int, error) { } func (i *KafkaInput) String() string { - return "Kafka Input: " + i.config.host + "/" + i.config.topic + return "Kafka Input: " + i.config.Host + "/" + i.config.Topic } diff --git a/input_kafka_test.go b/input_kafka_test.go index 9c0c7f1..2c6be17 100644 --- a/input_kafka_test.go +++ b/input_kafka_test.go @@ -16,10 +16,10 @@ func TestInputKafkaRAW(t *testing.T) { map[string][]int32{"test": {0}}, ) - input := NewKafkaInput("", &KafkaConfig{ + input := NewKafkaInput("", &InputKafkaConfig{ consumer: consumer, - topic: "test", - useJSON: false, + Topic: "test", + UseJSON: false, }) buf := make([]byte, 1024) @@ -43,10 +43,10 @@ func TestInputKafkaJSON(t *testing.T) { map[string][]int32{"test": {0}}, ) - input := NewKafkaInput("", &KafkaConfig{ + input := NewKafkaInput("", &InputKafkaConfig{ consumer: consumer, - topic: "test", - useJSON: true, + Topic: "test", + UseJSON: true, }) buf := make([]byte, 1024) diff --git a/input_raw.go b/input_raw.go index 3f1c60a..414ed9a 100644 --- a/input_raw.go +++ b/input_raw.go @@ -98,7 +98,7 @@ func (i *RAWInput) listen(address string) { log.Fatalf("input-raw: error while parsing address: %s", err) } - i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol, i.bpfFilter, i.timestampType, i.bufferSize, Settings.inputRAWOverrideSnapLen, Settings.inputRAWImmediateMode) + i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol, i.bpfFilter, i.timestampType, i.bufferSize, Settings.InputRAWOverrideSnapLen, Settings.InputRAWImmediateMode) ch := i.listener.Receiver() diff --git a/input_raw_test.go b/input_raw_test.go index 4a1114f..314997d 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -58,7 +58,7 @@ func TestRAWInputIPv4(t *testing.T) { atomic.AddInt64(&respCounter, 1) } - if Settings.debug { + if Settings.Debug { log.Println(reqCounter, respCounter) } @@ -74,7 +74,7 @@ func TestRAWInputIPv4(t *testing.T) { client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{}) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { // request + response @@ -125,7 +125,7 @@ func TestRAWInputNoKeepAlive(t *testing.T) { client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{}) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { // request + response @@ -168,7 +168,7 @@ func TestRAWInputIPv6(t *testing.T) { atomic.AddInt64(&respCounter, 1) } - if Settings.debug { + if Settings.Debug { log.Println(reqCounter, respCounter) } @@ -184,7 +184,7 @@ func TestRAWInputIPv6(t *testing.T) { client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{}) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { // request + response @@ -251,7 +251,7 @@ func TestInputRAW100Expect(t *testing.T) { plugins.All = append(plugins.All, input, testOutput, httpOutput) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) // Origin + Response/Request Test Output + Request Http Output wg.Add(4) @@ -305,7 +305,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { plugins.All = append(plugins.All, input, httpOutput) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) wg.Add(2) curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md") @@ -373,7 +373,7 @@ func TestInputRAWLargePayload(t *testing.T) { plugins.All = append(plugins.All, input, httpOutput) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) wg.Add(2) curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@/tmp/large") @@ -421,7 +421,7 @@ func BenchmarkRAWInput(b *testing.B) { plugins.All = append(plugins.All, input, output, httpOutput) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) emitted := 0 fileContent, _ := ioutil.ReadFile("LICENSE.txt") diff --git a/input_tcp.go b/input_tcp.go index 7a0eaf1..9472512 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -21,9 +21,9 @@ type TCPInput struct { } type TCPInputConfig struct { - secure bool - certificatePath string - keyPath string + secure bool `json:"input-tcp-secure"` + certificatePath string `json:"input-tcp-certificate"` + keyPath string `json:"input-tcp-certificate-key"` } // NewTCPInput constructor for TCPInput, accepts address with port diff --git a/input_tcp_test.go b/input_tcp_test.go index 5fd745d..def32d1 100644 --- a/input_tcp_test.go +++ b/input_tcp_test.go @@ -34,7 +34,7 @@ func TestTCPInput(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String()) @@ -117,7 +117,7 @@ func TestTCPInputSecure(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) conf := &tls.Config{ InsecureSkipVerify: true, diff --git a/kafka.go b/kafka.go index 7fe4d69..cc7ee58 100644 --- a/kafka.go +++ b/kafka.go @@ -10,12 +10,21 @@ import ( // KafkaConfig should contains required information to // build producers. -type KafkaConfig struct { - host string - topic string + +type InputKafkaConfig struct { producer sarama.AsyncProducer consumer sarama.Consumer - useJSON bool + Host string `json:"input-kafka-Host"` + Topic string `json:"input-kafka-Topic"` + UseJSON bool `json:"input-kafka-json-format"` +} + +type OutputKafkaConfig struct { + producer sarama.AsyncProducer + consumer sarama.Consumer + Host string `json:"output-kafka-Host"` + Topic string `json:"output-kafka-Topic"` + UseJSON bool `json:"output-kafka-json-format"` } // KafkaMessage should contains catched request information that should be diff --git a/limiter_test.go b/limiter_test.go index 1dafb9a..90d0174 100644 --- a/limiter_test.go +++ b/limiter_test.go @@ -25,7 +25,7 @@ func TestOutputLimiter(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { input.EmitGET() @@ -52,7 +52,7 @@ func TestInputLimiter(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { input.(*Limiter).plugin.(*TestInput).EmitGET() @@ -79,7 +79,7 @@ func TestPercentLimiter1(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { input.EmitGET() @@ -107,7 +107,7 @@ func TestPercentLimiter2(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { input.EmitGET() diff --git a/middleware.go b/middleware.go index 07eb3c7..000120b 100644 --- a/middleware.go +++ b/middleware.go @@ -75,7 +75,7 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { payload := buf[0:nr] - if Settings.prettifyHTTP { + if Settings.PrettifyHTTP { payload = prettifyHTTP(payload) nr = len(payload) @@ -84,7 +84,7 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { } } - if Settings.prettifyHTTP { + if Settings.PrettifyHTTP { payload = prettifyHTTP(payload) nr = len(payload) } @@ -96,7 +96,7 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { to.Write(dst[0 : nr*2+1]) m.mu.Unlock() - if Settings.debug { + if Settings.Debug { Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from) } } @@ -121,7 +121,7 @@ func (m *Middleware) read(from io.Reader) { fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line), string(line[:len(line)-1])) } - if Settings.debug { + if Settings.Debug { Debug("[MIDDLEWARE-MASTER] Received:", string(buf)) } diff --git a/middleware_test.go b/middleware_test.go index db94603..a7c4781 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -114,7 +114,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) - Settings.middleware = "./examples/middleware/echo.sh" + Settings.Middleware = "./examples/middleware/echo.sh" // Catch traffic from one service fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) @@ -132,7 +132,7 @@ func TestEchoMiddleware(t *testing.T) { // Start Gor emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) // Wait till middleware initialization time.Sleep(100 * time.Millisecond) @@ -153,7 +153,7 @@ func TestEchoMiddleware(t *testing.T) { emitter.Close() time.Sleep(200 * time.Millisecond) - Settings.middleware = "" + Settings.Middleware = "" } func TestTokenMiddleware(t *testing.T) { @@ -180,7 +180,7 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) - Settings.middleware = "go run ./examples/middleware/token_modifier.go" + Settings.Middleware = "go run ./examples/middleware/token_modifier.go" fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) // Catch traffic from one service @@ -198,7 +198,7 @@ func TestTokenMiddleware(t *testing.T) { // Start Gor emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) // Wait for middleware to initialize // Give go compiller time to build programm @@ -225,5 +225,5 @@ func TestTokenMiddleware(t *testing.T) { wg.Wait() emitter.Close() time.Sleep(100 * time.Millisecond) - Settings.middleware = "" + Settings.Middleware = "" } diff --git a/output_binary.go b/output_binary.go index e7eef6e..fe6152b 100644 --- a/output_binary.go +++ b/output_binary.go @@ -8,11 +8,11 @@ import ( // BinaryOutputConfig struct for holding binary output configuration type BinaryOutputConfig struct { - workers int - Timeout time.Duration - BufferSize int - Debug bool - TrackResponses bool + workers int `json:"output-binary-workers"` + Timeout time.Duration `json:"output-binary-timeout"` + BufferSize int `json:"output-tcp-response-buffer"` + Debug bool `json:"output-binary-debug"` + TrackResponses bool `json:"output-binary-track-response"` } // BinaryOutput plugin manage pool of workers which send request to replayed server diff --git a/output_file.go b/output_file.go index cd168db..3dd8925 100644 --- a/output_file.go +++ b/output_file.go @@ -31,12 +31,12 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{ // FileOutputConfig ... type FileOutputConfig struct { - flushInterval time.Duration + flushInterval time.Duration `json:"output-file-flush-interval"` sizeLimit int64 outputFileMaxSize int64 - queueLimit int64 - append bool - bufferPath string + queueLimit int64 `json:"output-file-queue-limit"` + append bool `json:"output-file-append"` + bufferPath string `json:"output-file-buffer"` onClose func(string) } @@ -233,7 +233,7 @@ func (o *FileOutput) Write(data []byte) (n int, err error) { o.totalFileSize += int64(n) o.queueLength++ - if Settings.outputFileConfig.outputFileMaxSize > 0 && o.totalFileSize >= Settings.outputFileConfig.outputFileMaxSize { + if Settings.OutputFileConfig.outputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.outputFileMaxSize { return n, errors.New("File output reached size limit") } diff --git a/output_file_test.go b/output_file_test.go index 2cfd903..2a05867 100644 --- a/output_file_test.go +++ b/output_file_test.go @@ -27,7 +27,7 @@ func TestFileOutput(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { wg.Add(2) @@ -53,7 +53,7 @@ func TestFileOutput(t *testing.T) { quit2 := make(chan int) emitter2 := NewEmitter(quit2) - go emitter2.Start(plugins2, Settings.middleware) + go emitter2.Start(plugins2, Settings.Middleware) wg.Wait() emitter2.Close() diff --git a/output_http.go b/output_http.go index 0ecb9d1..df29c52 100644 --- a/output_http.go +++ b/output_http.go @@ -62,28 +62,28 @@ type response struct { // HTTPOutputConfig struct for holding http output configuration type HTTPOutputConfig struct { - redirectLimit int + redirectLimit int `json:"output-http-redirects"` - stats bool - workersMin int - workersMax int - statsMs int + stats bool `json:"output-http-stats"` + workersMin int `json:"output-http-workers-min"` + workersMax int `json:"output-http-workers"` + statsMs int `json:"output-http-stats-ms"` workers int - queueLen int + queueLen int `json:"output-http-queue-len"` - elasticSearch string + elasticSearch string `json:"output-http-elasticsearch"` - Timeout time.Duration - OriginalHost bool - BufferSize int + Timeout time.Duration `json:"output-http-timeout"` + OriginalHost bool `json:"http-original-Host"` + BufferSize int `json:"output-http-response-buffer"` - CompatibilityMode bool + CompatibilityMode bool `json:"output-http-compatibility-mode"` RequestGroup string - Debug bool + Debug bool `json:"output-http-debug"` - TrackResponses bool + TrackResponses bool `json:"output-http-track-response"` } // HTTPOutput plugin manage pool of workers which send request to replayed server @@ -143,7 +143,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o.elasticSearch.Init(o.config.elasticSearch) } - if Settings.recognizeTCPSessions { + if Settings.RecognizeTCPSessions { if !PRO { log.Fatal("Detailed TCP sessions work only with PRO license") } @@ -249,7 +249,7 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { o.queueStats.Write(len(o.queue)) } - if !Settings.recognizeTCPSessions && o.config.workersMax != o.config.workersMin { + if !Settings.RecognizeTCPSessions && o.config.workersMax != o.config.workersMin { workersCount := int(atomic.LoadInt64(&o.activeWorkers)) if len(o.queue) > workersCount { @@ -275,7 +275,7 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { case resp = <-o.responses: } - if Settings.debug { + if Settings.Debug { Debug("[OUTPUT-HTTP] Received response:", string(resp.payload)) } @@ -289,7 +289,7 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { meta := payloadMeta(request) - if Settings.debug { + if Settings.Debug { Debug(meta) } diff --git a/output_http_test.go b/output_http_test.go index 9166708..af9272b 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -42,7 +42,7 @@ func TestHTTPOutput(t *testing.T) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")} - Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods} + Settings.ModifierConfig = HTTPModifierConfig{headers: headers, methods: methods} http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true, TrackResponses: true}) output := NewTestOutput(func(data []byte) { @@ -56,7 +56,7 @@ func TestHTTPOutput(t *testing.T) { plugins.All = append(plugins.All, input, output, http_output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 10; i++ { // 2 http-output, 2 - test output request, 2 - test output http response @@ -75,7 +75,7 @@ func TestHTTPOutput(t *testing.T) { t.Error("Should create workers for each request", activeWorkers) } - Settings.modifierConfig = HTTPModifierConfig{} + Settings.ModifierConfig = HTTPModifierConfig{} } func TestHTTPOutputKeepOriginalHost(t *testing.T) { @@ -85,7 +85,7 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) { input := NewTestInput() server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.Host != "custom-host.com" { + if req.Host != "custom-Host.com" { t.Error("Wrong header", req.Host) } @@ -93,8 +93,8 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) { })) defer server.Close() - headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}} - Settings.modifierConfig = HTTPModifierConfig{headers: headers} + headers := HTTPHeaders{HTTPHeader{"Host", "custom-Host.com"}} + Settings.ModifierConfig = HTTPModifierConfig{headers: headers} output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true}) @@ -105,14 +105,14 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) wg.Add(1) input.EmitGET() wg.Wait() emitter.Close() - Settings.modifierConfig = HTTPModifierConfig{} + Settings.ModifierConfig = HTTPModifierConfig{} } func TestHTTPOutputSSL(t *testing.T) { @@ -134,7 +134,7 @@ func TestHTTPOutputSSL(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) wg.Add(2) @@ -157,7 +157,7 @@ func TestHTTPOutputSessions(t *testing.T) { })) defer server.Close() - Settings.recognizeTCPSessions = true + Settings.RecognizeTCPSessions = true output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true}) plugins := &InOutPlugins{ @@ -165,7 +165,7 @@ func TestHTTPOutputSessions(t *testing.T) { Outputs: []io.Writer{output}, } emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) uuid1 := []byte("1234567890123456789a0000") uuid2 := []byte("1234567890123456789d0000") @@ -190,7 +190,7 @@ func TestHTTPOutputSessions(t *testing.T) { emitter.Close() - Settings.recognizeTCPSessions = false + Settings.RecognizeTCPSessions = false } func BenchmarkHTTPOutput(b *testing.B) { @@ -213,7 +213,7 @@ func BenchmarkHTTPOutput(b *testing.B) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < b.N; i++ { wg.Add(1) diff --git a/output_kafka.go b/output_kafka.go index 07b35b6..6616a88 100644 --- a/output_kafka.go +++ b/output_kafka.go @@ -15,7 +15,7 @@ import ( // KafkaOutput is used for sending payloads to kafka in JSON format. type KafkaOutput struct { - config *KafkaConfig + config *OutputKafkaConfig producer sarama.AsyncProducer } @@ -23,7 +23,7 @@ type KafkaOutput struct { const KafkaOutputFrequency = 500 // NewKafkaOutput creates instance of kafka producer client. -func NewKafkaOutput(address string, config *KafkaConfig) io.Writer { +func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer { c := sarama.NewConfig() var producer sarama.AsyncProducer @@ -35,7 +35,7 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer { c.Producer.Compression = sarama.CompressionSnappy c.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond - brokerList := strings.Split(config.host, ",") + brokerList := strings.Split(config.Host, ",") var err error producer, err = sarama.NewAsyncProducer(brokerList, c) @@ -49,7 +49,7 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer { producer: producer, } - if Settings.verbose { + if Settings.Verbose { // Start infinite loop for tracking errors for kafka producer. go o.ErrorHandler() } @@ -67,7 +67,7 @@ func (o *KafkaOutput) ErrorHandler() { func (o *KafkaOutput) Write(data []byte) (n int, err error) { var message sarama.StringEncoder - if !o.config.useJSON { + if !o.config.UseJSON { message = sarama.StringEncoder(data) } else { headers := make(map[string]string) @@ -93,7 +93,7 @@ func (o *KafkaOutput) Write(data []byte) (n int, err error) { } o.producer.Input() <- &sarama.ProducerMessage{ - Topic: o.config.topic, + Topic: o.config.Topic, Value: message, } diff --git a/output_kafka_test.go b/output_kafka_test.go index 30a58e0..cc3cfc7 100644 --- a/output_kafka_test.go +++ b/output_kafka_test.go @@ -13,10 +13,10 @@ func TestOutputKafkaRAW(t *testing.T) { producer := mocks.NewAsyncProducer(t, config) producer.ExpectInputAndSucceed() - output := NewKafkaOutput("", &KafkaConfig{ + output := NewKafkaOutput("", &OutputKafkaConfig{ producer: producer, - topic: "test", - useJSON: false, + Topic: "test", + UseJSON: false, }) output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n")) @@ -36,10 +36,10 @@ func TestOutputKafkaJSON(t *testing.T) { producer := mocks.NewAsyncProducer(t, config) producer.ExpectInputAndSucceed() - output := NewKafkaOutput("", &KafkaConfig{ + output := NewKafkaOutput("", &OutputKafkaConfig{ producer: producer, - topic: "test", - useJSON: true, + Topic: "test", + UseJSON: true, }) output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n")) diff --git a/output_tcp.go b/output_tcp.go index d74d44c..6cc1c76 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -22,8 +22,8 @@ type TCPOutput struct { } type TCPOutputConfig struct { - secure bool - sticky bool + secure bool `json:"output-tcp-secure"` + sticky bool `json:"output-tcp-sticky"` } // NewTCPOutput constructor for TCPOutput @@ -34,7 +34,7 @@ func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer { o.address = address o.config = config - if Settings.outputTCPStats { + if Settings.OutputTCPStats { o.bufStats = NewGorStat("output_tcp", 5000) } @@ -114,7 +114,7 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) { bufferIndex := o.getBufferIndex(data) o.buf[bufferIndex] <- newBuf - if Settings.outputTCPStats { + if Settings.OutputTCPStats { o.bufStats.Write(len(o.buf[bufferIndex])) } diff --git a/output_tcp_test.go b/output_tcp_test.go index cf3fd90..1c82130 100644 --- a/output_tcp_test.go +++ b/output_tcp_test.go @@ -27,7 +27,7 @@ func TestTCPOutput(t *testing.T) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { wg.Add(1) @@ -82,7 +82,7 @@ func BenchmarkTCPOutput(b *testing.B) { plugins.All = append(plugins.All, input, output) emitter := NewEmitter(quit) - go emitter.Start(plugins, Settings.middleware) + go emitter.Start(plugins, Settings.Middleware) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/plugins.go b/plugins.go index 31b4cc8..da265f1 100644 --- a/plugins.go +++ b/plugins.go @@ -82,80 +82,80 @@ func InitPlugins() *InOutPlugins { pluginMu.Lock() defer pluginMu.Unlock() - for _, options := range Settings.inputDummy { + for _, options := range Settings.InputDummy { registerPlugin(NewDummyInput, options) } - for range Settings.outputDummy { + for range Settings.OutputDummy { registerPlugin(NewDummyOutput) } - if Settings.outputStdout { + if Settings.OutputStdout { registerPlugin(NewDummyOutput) } - if Settings.outputNull { + if Settings.OutputNull { registerPlugin(NewNullOutput) } engine := EnginePcap - if Settings.inputRAWEngine == "raw_socket" { + if Settings.InputRAWEngine == "raw_socket" { engine = EngineRawSocket - } else if Settings.inputRAWEngine == "pcap_file" { + } else if Settings.InputRAWEngine == "pcap_file" { engine = EnginePcapFile } - for _, options := range Settings.inputRAW { - registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, Settings.inputRAWExpire, Settings.inputRAWRealIPHeader, Settings.inputRAWProtocol, Settings.inputRAWBpfFilter, Settings.inputRAWTimestampType, Settings.inputRAWBufferSize) + for _, options := range Settings.InputRAW { + registerPlugin(NewRAWInput, options, engine, Settings.InputRAWTrackResponse, Settings.InputRAWExpire, Settings.InputRAWRealIPHeader, Settings.InputRAWProtocol, Settings.InputRAWBpfFilter, Settings.InputRAWTimestampType, Settings.inputRAWBufferSize) } - for _, options := range Settings.inputTCP { - registerPlugin(NewTCPInput, options, &Settings.inputTCPConfig) + for _, options := range Settings.InputTCP { + registerPlugin(NewTCPInput, options, &Settings.InputTCPConfig) } - for _, options := range Settings.outputTCP { - registerPlugin(NewTCPOutput, options, &Settings.outputTCPConfig) + for _, options := range Settings.OutputTCP { + registerPlugin(NewTCPOutput, options, &Settings.OutputTCPConfig) } - for _, options := range Settings.inputFile { - registerPlugin(NewFileInput, options, Settings.inputFileLoop) + for _, options := range Settings.InputFile { + registerPlugin(NewFileInput, options, Settings.InputFileLoop) } - for _, path := range Settings.outputFile { + for _, path := range Settings.OutputFile { if strings.HasPrefix(path, "s3://") { - registerPlugin(NewS3Output, path, &Settings.outputFileConfig) + registerPlugin(NewS3Output, path, &Settings.OutputFileConfig) } else { - registerPlugin(NewFileOutput, path, &Settings.outputFileConfig) + registerPlugin(NewFileOutput, path, &Settings.OutputFileConfig) } } - for _, options := range Settings.inputHTTP { + for _, options := range Settings.InputHTTP { registerPlugin(NewHTTPInput, options) } // If we explicitly set Host header http output should not rewrite it // Fix: https://github.com/buger/gor/issues/174 - for _, header := range Settings.modifierConfig.headers { + for _, header := range Settings.ModifierConfig.headers { if header.Name == "Host" { - Settings.outputHTTPConfig.OriginalHost = true + Settings.OutputHTTPConfig.OriginalHost = true break } } - for _, options := range Settings.outputHTTP { - registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig) + for _, options := range Settings.OutputHTTP { + registerPlugin(NewHTTPOutput, options, &Settings.OutputHTTPConfig) } - for _, options := range Settings.outputBinary { - registerPlugin(NewBinaryOutput, options, &Settings.outputBinaryConfig) + for _, options := range Settings.OutputBinary { + registerPlugin(NewBinaryOutput, options, &Settings.OutputBinaryConfig) } - if Settings.outputKafkaConfig.host != "" && Settings.outputKafkaConfig.topic != "" { - registerPlugin(NewKafkaOutput, "", &Settings.outputKafkaConfig) + if Settings.OutputKafkaConfig.Host != "" && Settings.OutputKafkaConfig.Topic != "" { + registerPlugin(NewKafkaOutput, "", &Settings.OutputKafkaConfig) } - if Settings.inputKafkaConfig.host != "" && Settings.inputKafkaConfig.topic != "" { - registerPlugin(NewKafkaInput, "", &Settings.inputKafkaConfig) + if Settings.InputKafkaConfig.Host != "" && Settings.InputKafkaConfig.Topic != "" { + registerPlugin(NewKafkaInput, "", &Settings.InputKafkaConfig) } return plugins diff --git a/plugins_test.go b/plugins_test.go index 6331ca0..0cfbb97 100644 --- a/plugins_test.go +++ b/plugins_test.go @@ -5,10 +5,10 @@ import ( ) func TestPluginsRegistration(t *testing.T) { - Settings.inputDummy = MultiOption{"[]"} - Settings.outputDummy = MultiOption{"[]"} - Settings.outputHTTP = MultiOption{"www.example.com|10"} - Settings.inputFile = MultiOption{"/dev/null"} + Settings.InputDummy = MultiOption{"[]"} + Settings.OutputDummy = MultiOption{"[]"} + Settings.OutputHTTP = MultiOption{"www.example.com|10"} + Settings.InputFile = MultiOption{"/dev/null"} plugins := InitPlugins() diff --git a/settings.go b/settings.go index 11c0a8f..9768944 100644 --- a/settings.go +++ b/settings.go @@ -29,64 +29,64 @@ func (h *MultiOption) Set(value string) error { // AppSettings is the struct of main configuration type AppSettings struct { - verbose bool - debug bool - stats bool - exitAfter time.Duration + Verbose bool `json:"verbose"` + Debug bool `json:"debug"` + Stats bool `json:"stats"` + ExitAfter time.Duration `json:"exit-after"` - splitOutput bool - recognizeTCPSessions bool - pprof string + SplitOutput bool `json:"split-output"` + RecognizeTCPSessions bool `json:"recognize-tcp-sessions"` + Pprof string `json:"http-pprof"` - inputDummy MultiOption - outputDummy MultiOption - outputStdout bool - outputNull bool + InputDummy MultiOption `json:"input-dummy"` + OutputDummy MultiOption `json:"output-dummy"` + OutputStdout bool `json:"output-stdout"` + OutputNull bool `json:"output-null"` - inputTCP MultiOption - inputTCPConfig TCPInputConfig - outputTCP MultiOption - outputTCPConfig TCPOutputConfig - outputTCPStats bool + InputTCP MultiOption `json:"input-tcp"` + InputTCPConfig TCPInputConfig `json:"input-tcp"` + OutputTCP MultiOption `json:"output-tcp"` + OutputTCPConfig TCPOutputConfig `json:"output-tcp"` + OutputTCPStats bool `json:"output-tcp-stats"` - inputFile MultiOption - inputFileLoop bool - outputFile MultiOption - outputFileConfig FileOutputConfig + InputFile MultiOption `json:"input-file"` + InputFileLoop bool `json:"input-file-loop"` + OutputFile MultiOption `json:"output-file"` + OutputFileConfig FileOutputConfig - inputRAW MultiOption - inputRAWEngine string - inputRAWTrackResponse bool - inputRAWRealIPHeader string - inputRAWExpire time.Duration - inputRAWProtocol string - inputRAWBpfFilter string - inputRAWTimestampType string + InputRAW MultiOption `json:"input-raw"` + InputRAWEngine string `json:"input-raw-engine"` + InputRAWTrackResponse bool `json:"input-raw-track-response"` + InputRAWRealIPHeader string `json:"input-raw-realip-header"` + InputRAWExpire time.Duration `json:"input-raw-expire"` + InputRAWProtocol string `json:"input-raw-protocol"` + InputRAWBpfFilter string `json:"input-raw-bpf-filter"` + InputRAWTimestampType string `json:"input-raw-timestamp-type"` copyBufferSize int64 - inputRAWImmediateMode bool + InputRAWImmediateMode bool `json:"input-raw-immediate-mode"` inputRAWBufferSize int64 - inputRAWOverrideSnapLen bool + InputRAWOverrideSnapLen bool `json:"input-raw-override-snaplen"` - inputRAWBufferSizeFlag string - outputFileSizeFlag string - outputFileMaxSizeFlag string - copyBufferSizeFlag string + InputRAWBufferSizeFlag string `json:"input-raw-buffer-size"` + OutputFileSizeFlag string `json:"output-file-size-limit"` + OutputFileMaxSizeFlag string `json:"output-file-max-size-limit"` + CopyBufferSizeFlag string `json:"copy-buffer-size"` - middleware string + Middleware string `json:"middleware"` - inputHTTP MultiOption - outputHTTP MultiOption - prettifyHTTP bool + InputHTTP MultiOption `json:"input-http"` + OutputHTTP MultiOption `json:"output-http"` + PrettifyHTTP bool `json:"prettify-http"` - outputHTTPConfig HTTPOutputConfig + OutputHTTPConfig HTTPOutputConfig - outputBinary MultiOption - outputBinaryConfig BinaryOutputConfig + OutputBinary MultiOption `json:"output-binary"` + OutputBinaryConfig BinaryOutputConfig - modifierConfig HTTPModifierConfig + ModifierConfig HTTPModifierConfig `json:"debug"` - inputKafkaConfig KafkaConfig - outputKafkaConfig KafkaConfig + InputKafkaConfig InputKafkaConfig + OutputKafkaConfig OutputKafkaConfig } // Settings holds Gor configuration @@ -101,181 +101,181 @@ func usage() { func init() { flag.Usage = usage - flag.StringVar(&Settings.pprof, "http-pprof", "", "Enable profiling. Starts http server on specified port, exposing special /debug/pprof endpoint. Example: `:8181`") - flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output") - flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag") - flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output") + flag.StringVar(&Settings.Pprof, "http-pprof", "", "Enable profiling. Starts http server on specified port, exposing special /debug/pprof endpoint. Example: `:8181`") + flag.BoolVar(&Settings.Verbose, "verbose", false, "Turn on more verbose output") + flag.BoolVar(&Settings.Debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag") + flag.BoolVar(&Settings.Stats, "stats", false, "Turn on queue stats output") if DEMO == "" { - flag.DurationVar(&Settings.exitAfter, "exit-after", 0, "exit after specified duration") + flag.DurationVar(&Settings.ExitAfter, "exit-after", 0, "exit after specified duration") } else { - Settings.exitAfter = 5 * time.Minute + Settings.ExitAfter = 5 * time.Minute } - flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.") + flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.") - flag.BoolVar(&Settings.recognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.") + flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.") - flag.Var(&Settings.inputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s") - flag.Var(&Settings.outputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead") + flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s") + flag.Var(&Settings.OutputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead") - flag.BoolVar(&Settings.outputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.") + flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.") - flag.BoolVar(&Settings.outputNull, "output-null", false, "Used for testing inputs. Drops all requests.") + flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.") - flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com") - flag.BoolVar(&Settings.inputTCPConfig.secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.") - flag.StringVar(&Settings.inputTCPConfig.certificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.") - flag.StringVar(&Settings.inputTCPConfig.keyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.") + flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com") + flag.BoolVar(&Settings.InputTCPConfig.secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.") + flag.StringVar(&Settings.InputTCPConfig.certificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.") + flag.StringVar(&Settings.InputTCPConfig.keyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.") - flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020") - flag.BoolVar(&Settings.outputTCPConfig.secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.") - flag.BoolVar(&Settings.outputTCPConfig.sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.") - flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.") + flag.Var(&Settings.OutputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020") + flag.BoolVar(&Settings.OutputTCPConfig.secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.") + flag.BoolVar(&Settings.OutputTCPConfig.sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.") + flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.") - flag.Var(&Settings.inputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com") - flag.BoolVar(&Settings.inputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.") + flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com") + flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.") - flag.Var(&Settings.outputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor") - flag.DurationVar(&Settings.outputFileConfig.flushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.") - flag.BoolVar(&Settings.outputFileConfig.append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ") - flag.StringVar(&Settings.outputFileSizeFlag, "output-file-size-limit", "32mb", "Size of each chunk. Default: 32mb") - flag.Int64Var(&Settings.outputFileConfig.queueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256") - flag.StringVar(&Settings.outputFileMaxSizeFlag, "output-file-max-size-limit", "1TB", "Max size of output file, Default: 1TB") + flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor") + flag.DurationVar(&Settings.OutputFileConfig.flushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.") + flag.BoolVar(&Settings.OutputFileConfig.append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ") + flag.StringVar(&Settings.OutputFileSizeFlag, "output-file-size-limit", "32mb", "Size of each chunk. Default: 32mb") + flag.Int64Var(&Settings.OutputFileConfig.queueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256") + flag.StringVar(&Settings.OutputFileMaxSizeFlag, "output-file-max-size-limit", "1TB", "Max size of output file, Default: 1TB") - flag.StringVar(&Settings.outputFileConfig.bufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs") + flag.StringVar(&Settings.OutputFileConfig.bufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs") - flag.BoolVar(&Settings.prettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encodning: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout") + flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encodning: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout") - flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") + flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") - flag.BoolVar(&Settings.inputRAWTrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.") + flag.BoolVar(&Settings.InputRAWTrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.") - flag.StringVar(&Settings.inputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`") + flag.StringVar(&Settings.InputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`") - flag.StringVar(&Settings.inputRAWProtocol, "input-raw-protocol", "http", "Specify application protocol of intercepted traffic. Possible values: http, binary") + flag.StringVar(&Settings.InputRAWProtocol, "input-raw-protocol", "http", "Specify application protocol of intercepted traffic. Possible values: http, binary") - flag.StringVar(&Settings.inputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP") + flag.StringVar(&Settings.InputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP") - flag.DurationVar(&Settings.inputRAWExpire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.") + flag.DurationVar(&Settings.InputRAWExpire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.") - flag.StringVar(&Settings.inputRAWBpfFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'") + flag.StringVar(&Settings.InputRAWBpfFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'") - flag.StringVar(&Settings.inputRAWTimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.") - flag.StringVar(&Settings.copyBufferSizeFlag, "copy-buffer-size", "5mb", "Set the buffer size for an individual request (default 5MB)") - flag.BoolVar(&Settings.inputRAWOverrideSnapLen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments") - flag.BoolVar(&Settings.inputRAWImmediateMode, "input-raw-immediate-mode", false, "Set pcap interface to immediate mode.") - flag.StringVar(&Settings.inputRAWBufferSizeFlag, "input-raw-buffer-size", "0", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.") + flag.StringVar(&Settings.InputRAWTimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.") + flag.StringVar(&Settings.CopyBufferSizeFlag, "copy-buffer-size", "5mb", "Set the buffer size for an individual request (default 5MB)") + flag.BoolVar(&Settings.InputRAWOverrideSnapLen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments") + flag.BoolVar(&Settings.InputRAWImmediateMode, "input-raw-immediate-mode", false, "Set pcap interface to immediate mode.") + flag.StringVar(&Settings.InputRAWBufferSizeFlag, "input-raw-buffer-size", "0", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.") - flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command") + flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command") // flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") - flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") + flag.Var(&Settings.OutputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") /* outputHTTPConfig */ - flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") - flag.BoolVar(&Settings.outputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.") + flag.IntVar(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") + flag.BoolVar(&Settings.OutputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.") - flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.") - flag.IntVar(&Settings.outputHTTPConfig.workersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.") - flag.IntVar(&Settings.outputHTTPConfig.queueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000") + flag.IntVar(&Settings.OutputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.") + flag.IntVar(&Settings.OutputHTTPConfig.workersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.") + flag.IntVar(&Settings.OutputHTTPConfig.queueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000") - flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") - flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") - flag.BoolVar(&Settings.outputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.") + flag.IntVar(&Settings.OutputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") + flag.DurationVar(&Settings.OutputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") + flag.BoolVar(&Settings.OutputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.") - flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every N milliseconds. See output-http-stats-ms") - flag.IntVar(&Settings.outputHTTPConfig.statsMs, "output-http-stats-ms", 5000, "Report http output queue stats to console every N milliseconds. default: 5000") - flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.") - flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.") - flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") + flag.BoolVar(&Settings.OutputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every N milliseconds. See output-http-stats-ms") + flag.IntVar(&Settings.OutputHTTPConfig.statsMs, "output-http-stats-ms", 5000, "Report http output queue stats to console every N milliseconds. default: 5000") + flag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, "http-original-Host", false, "Normally gor replaces the Host http header with the Host supplied with --output-http. This option disables that behavior, preserving the original Host header.") + flag.BoolVar(&Settings.OutputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.") + flag.StringVar(&Settings.OutputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") /* outputHTTPConfig */ - flag.Var(&Settings.outputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80") + flag.Var(&Settings.OutputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80") /* outputBinaryConfig */ - flag.IntVar(&Settings.outputBinaryConfig.BufferSize, "output-tcp-response-buffer", 0, "TCP response buffer size, all data after this size will be discarded.") - flag.IntVar(&Settings.outputBinaryConfig.workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") - flag.DurationVar(&Settings.outputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s") - flag.BoolVar(&Settings.outputBinaryConfig.TrackResponses, "output-binary-track-response", false, "If turned on, Binary output responses will be set to all outputs like stdout, file and etc.") + flag.IntVar(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", 0, "TCP response buffer size, all data after this size will be discarded.") + flag.IntVar(&Settings.OutputBinaryConfig.workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") + flag.DurationVar(&Settings.OutputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s") + flag.BoolVar(&Settings.OutputBinaryConfig.TrackResponses, "output-binary-track-response", false, "If turned on, Binary output responses will be set to all outputs like stdout, file and etc.") - flag.BoolVar(&Settings.outputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.") + flag.BoolVar(&Settings.OutputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.") /* outputBinaryConfig */ - flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'") - flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'") - flag.BoolVar(&Settings.outputKafkaConfig.useJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.") + flag.StringVar(&Settings.OutputKafkaConfig.Host, "output-kafka-Host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-Host '192.168.0.1:9092,192.168.0.2:9092'") + flag.StringVar(&Settings.OutputKafkaConfig.Topic, "output-kafka-Topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-Topic 'kafka-log'") + flag.BoolVar(&Settings.OutputKafkaConfig.UseJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.") - flag.StringVar(&Settings.inputKafkaConfig.host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'") - flag.StringVar(&Settings.inputKafkaConfig.topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'") - flag.BoolVar(&Settings.inputKafkaConfig.useJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than GoReplay text format.") + flag.StringVar(&Settings.InputKafkaConfig.Host, "input-kafka-Host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-Host '192.168.0.1:9092,192.168.0.2:9092'") + flag.StringVar(&Settings.InputKafkaConfig.Topic, "input-kafka-Topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-Topic 'kafka-log'") + flag.BoolVar(&Settings.InputKafkaConfig.UseJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than GoReplay text format.") - flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") - flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead") + flag.Var(&Settings.ModifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") + flag.Var(&Settings.ModifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead") - flag.Var(&Settings.modifierConfig.headerRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com") + flag.Var(&Settings.ModifierConfig.headerRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com") - flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1") + flag.Var(&Settings.ModifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1") - flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS") - flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead") + flag.Var(&Settings.ModifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS") + flag.Var(&Settings.ModifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead") - flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.") - flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead") + flag.Var(&Settings.ModifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.") + flag.Var(&Settings.ModifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead") - flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.") + flag.Var(&Settings.ModifierConfig.urlNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.") - flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") - flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead") + flag.Var(&Settings.ModifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + flag.Var(&Settings.ModifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead") - flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") - flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") + flag.Var(&Settings.ModifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") + flag.Var(&Settings.ModifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") - flag.Var(&Settings.modifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"") + flag.Var(&Settings.ModifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"") - flag.Var(&Settings.modifierConfig.headerBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"") + flag.Var(&Settings.ModifierConfig.headerBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"") - flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%") + flag.Var(&Settings.ModifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%") - flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead") + flag.Var(&Settings.ModifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead") - flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%") + flag.Var(&Settings.ModifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%") // default values, using for tests - Settings.outputFileConfig.sizeLimit = 33554432 - Settings.outputFileConfig.outputFileMaxSize = 1099511627776 + Settings.OutputFileConfig.sizeLimit = 33554432 + Settings.OutputFileConfig.outputFileMaxSize = 1099511627776 Settings.copyBufferSize = 5242880 Settings.inputRAWBufferSize = 0 } func checkSettings() { - outputFileSize, err := bufferParser(Settings.outputFileSizeFlag, "32MB") + outputFileSize, err := bufferParser(Settings.OutputFileSizeFlag, "32MB") if err != nil { log.Fatalf("output-file-size-limit error: %v\n", err) } - Settings.outputFileConfig.sizeLimit = outputFileSize + Settings.OutputFileConfig.sizeLimit = outputFileSize - outputFileMaxSize, err := bufferParser(Settings.outputFileMaxSizeFlag, "1TB") + outputFileMaxSize, err := bufferParser(Settings.OutputFileMaxSizeFlag, "1TB") if err != nil { log.Fatalf("output-file-max-size-limit error: %v\n", err) } - Settings.outputFileConfig.outputFileMaxSize = outputFileMaxSize + Settings.OutputFileConfig.outputFileMaxSize = outputFileMaxSize - copyBufferSize, err := bufferParser(Settings.copyBufferSizeFlag, "5mb") + copyBufferSize, err := bufferParser(Settings.CopyBufferSizeFlag, "5mb") if err != nil { log.Fatalf("copy-buffer-size error: %v\n", err) } Settings.copyBufferSize = copyBufferSize - inputRAWBufferSize, err := bufferParser(Settings.inputRAWBufferSizeFlag, "0") + inputRAWBufferSize, err := bufferParser(Settings.InputRAWBufferSizeFlag, "0") if err != nil { log.Fatalf("input-raw-buffer-size error: %v\n", err) } Settings.inputRAWBufferSize = inputRAWBufferSize // libpcap has bug in mac os x. More info: https://github.com/buger/goreplay/issues/730 - if Settings.inputRAWExpire == time.Second*2 && runtime.GOOS == "darwin" { - Settings.inputRAWExpire = time.Second + if Settings.InputRAWExpire == time.Second*2 && runtime.GOOS == "darwin" { + Settings.InputRAWExpire = time.Second } } @@ -285,7 +285,7 @@ var pID = os.Getpid() // Debug take an effect only if --verbose flag specified func Debug(args ...interface{}) { - if Settings.verbose { + if Settings.Verbose { debugMutex.Lock() defer debugMutex.Unlock() now := time.Now() diff --git a/settings_test.go b/settings_test.go new file mode 100644 index 0000000..bd466ce --- /dev/null +++ b/settings_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "encoding/json" + "fmt" + "testing" +) + +func TestAppSettings(t *testing.T) { + var a AppSettings + data, err := json.Marshal(a) + if err != nil { + panic(err) + } + fmt.Printf(string(data)) +} diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index 710d551..4eab77c 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -684,11 +684,16 @@ var awsPartition = partition{ "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, + "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, @@ -2949,6 +2954,12 @@ var awsPartition = partition{ "us-east-1": endpoint{}, }, }, + "honeycode": service{ + + Endpoints: endpoints{ + "us-west-2": endpoint{}, + }, + }, "iam": service{ PartitionEndpoint: "aws-global", IsRegionalized: boxedFalse, @@ -3473,12 +3484,36 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "me-south-1": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "logs-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "logs-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "logs-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "logs-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, "machinelearning": service{ @@ -3699,6 +3734,7 @@ var awsPartition = partition{ }, }, Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "eu-central-1": endpoint{}, @@ -4026,9 +4062,11 @@ var awsPartition = partition{ "outposts": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, @@ -4068,6 +4106,7 @@ var awsPartition = partition{ }, }, "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, @@ -4246,6 +4285,7 @@ var awsPartition = partition{ "ram": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, @@ -4255,6 +4295,7 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, @@ -4507,6 +4548,7 @@ var awsPartition = partition{ }, }, Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "eu-central-1": endpoint{}, @@ -6649,6 +6691,25 @@ var awscnPartition = partition{ }, }, }, + "organizations": service{ + PartitionEndpoint: "aws-cn-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-cn-global": endpoint{ + Hostname: "organizations.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + "fips-aws-cn-global": endpoint{ + Hostname: "organizations.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, "polly": service{ Endpoints: endpoints{ @@ -7073,6 +7134,13 @@ var awsusgovPartition = partition{ "us-gov-west-1": endpoint{}, }, }, + "backup": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, "batch": service{ Endpoints: endpoints{ @@ -7306,6 +7374,17 @@ var awsusgovPartition = partition{ "us-gov-west-1": endpoint{}, }, }, + "docdb": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{ + Hostname: "rds.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, "ds": service{ Endpoints: endpoints{ @@ -7704,6 +7783,13 @@ var awsusgovPartition = partition{ "us-gov-west-1": endpoint{}, }, }, + "kinesisanalytics": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, "kms": service{ Endpoints: endpoints{ @@ -7758,8 +7844,18 @@ var awsusgovPartition = partition{ "logs": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-gov-east-1": endpoint{ + Hostname: "logs.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "logs.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, "mediaconvert": service{ diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index c050612..eab2596 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.32.7" +const SDKVersion = "1.33.2" diff --git a/vendor/modules.txt b/vendor/modules.txt index d5c6868..ea153c7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -5,7 +5,7 @@ github.com/Shopify/sarama/mocks # github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 ## explicit github.com/araddon/gou -# github.com/aws/aws-sdk-go v1.32.7 +# github.com/aws/aws-sdk-go v1.33.2 ## explicit github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/arn @@ -102,7 +102,7 @@ github.com/rcrowley/go-metrics ## explicit golang.org/x/crypto/md4 golang.org/x/crypto/pbkdf2 -# golang.org/x/net v0.0.0-20200602114024-627f9648deb9 +# golang.org/x/net v0.0.0-20200707034311-ab3426394381 ## explicit golang.org/x/net/internal/socks golang.org/x/net/proxy