mirror of
https://github.com/buger/goreplay.git
synced 2024-04-21 12:32:02 +00:00
Urban IshimweandGitHub
5d8ca525a4
for easy readability check changes by their underlying commits! these benchmarks address the whole operation of the request cycle in goreplay. _**goos: linux goarch: amd64**_ **Using Compatible client:** ``` BenchmarkHTTPOutput-4 10417 118969 ns/op 12172 B/op 93 allocs/op BenchmarkHTTPOutputTLS-4 9136 132929 ns/op 12448 B/op 97 allocs/op ``` **Using non-compatible client** ``` BenchmarkHTTPOutput-4 859 1175040 ns/op 15598 B/op 46 allocs/op BenchmarkHTTPOutputTLS-4 880 1189643 ns/op 15544 B/op 52 allocs/op ``` Binary size reduced: **7%** from these benchmarks, we may trade allocations with performance and memory!
106 lines
2.7 KiB
Go
106 lines
2.7 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"log"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/buger/goreplay/byteutils"
|
|
"github.com/buger/goreplay/proto"
|
|
|
|
"github.com/Shopify/sarama"
|
|
"github.com/Shopify/sarama/mocks"
|
|
)
|
|
|
|
// KafkaOutput is used for sending payloads to kafka in JSON format.
|
|
type KafkaOutput struct {
|
|
config *OutputKafkaConfig
|
|
producer sarama.AsyncProducer
|
|
}
|
|
|
|
// KafkaOutputFrequency in milliseconds
|
|
const KafkaOutputFrequency = 500
|
|
|
|
// NewKafkaOutput creates instance of kafka producer client.
|
|
func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
|
|
return NewKafkaOutputWithTLS(address, config, nil)
|
|
}
|
|
|
|
// NewKafkaOutputWithTLS creates instance of kafka producer client.
|
|
func NewKafkaOutputWithTLS(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) io.Writer {
|
|
c := NewKafkaConfig(tlsConfig)
|
|
|
|
var producer sarama.AsyncProducer
|
|
|
|
if mock, ok := config.producer.(*mocks.AsyncProducer); ok && mock != nil {
|
|
producer = config.producer
|
|
} else {
|
|
c.Producer.RequiredAcks = sarama.WaitForLocal
|
|
c.Producer.Compression = sarama.CompressionSnappy
|
|
c.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond
|
|
|
|
brokerList := strings.Split(config.Host, ",")
|
|
|
|
var err error
|
|
producer, err = sarama.NewAsyncProducer(brokerList, c)
|
|
if err != nil {
|
|
log.Fatalln("Failed to start Sarama(Kafka) producer:", err)
|
|
}
|
|
}
|
|
|
|
o := &KafkaOutput{
|
|
config: config,
|
|
producer: producer,
|
|
}
|
|
|
|
// Start infinite loop for tracking errors for kafka producer.
|
|
go o.ErrorHandler()
|
|
|
|
return o
|
|
}
|
|
|
|
// ErrorHandler should receive errors
|
|
func (o *KafkaOutput) ErrorHandler() {
|
|
for err := range o.producer.Errors() {
|
|
Debug(1, "Failed to write access log entry:", err)
|
|
}
|
|
}
|
|
|
|
func (o *KafkaOutput) Write(data []byte) (n int, err error) {
|
|
var message sarama.StringEncoder
|
|
|
|
if !o.config.UseJSON {
|
|
message = sarama.StringEncoder(byteutils.SliceToString(data))
|
|
} else {
|
|
mimeHeader := proto.ParseHeaders(data)
|
|
var header map[string]string
|
|
for k, v := range mimeHeader {
|
|
header[k] = strings.Join(v, ", ")
|
|
}
|
|
|
|
meta := payloadMeta(data)
|
|
req := payloadBody(data)
|
|
|
|
kafkaMessage := KafkaMessage{
|
|
ReqURL: byteutils.SliceToString(proto.Path(req)),
|
|
ReqType: byteutils.SliceToString(meta[0]),
|
|
ReqID: byteutils.SliceToString(meta[1]),
|
|
ReqTs: byteutils.SliceToString(meta[2]),
|
|
ReqMethod: byteutils.SliceToString(proto.Method(req)),
|
|
ReqBody: byteutils.SliceToString(proto.Body(req)),
|
|
ReqHeaders: header,
|
|
}
|
|
jsonMessage, _ := json.Marshal(&kafkaMessage)
|
|
message = sarama.StringEncoder(byteutils.SliceToString(jsonMessage))
|
|
}
|
|
|
|
o.producer.Input() <- &sarama.ProducerMessage{
|
|
Topic: o.config.Topic,
|
|
Value: message,
|
|
}
|
|
|
|
return len(message), nil
|
|
}
|