github/goreplay
4
0
Fork 0
mirror of https://github.com/buger/goreplay.git synced 2024-04-21 12:32:02 +00:00
Code Issues Releases Wiki Activity

Update elasticsearch output to use vendored elastigo and fix #331. (#333)

Browse Source 
* add elastigo submodule

* add gou submodule

* add go-hostpool submodule

* use vendored elasticgo

* update ReqUrl to ReqURL
This commit is contained in:
Joseph Lawson
2016-07-07 16:30:51 +03:00
committed by Leonid Bugaev
parent c128d46088
commit e65035dd95
5 changed files with 64 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitmodules
+9
View File
@@ -0,0 +1,9 @@
[submodule "vendor/github.com/mattbaird/elastigo"]
path = vendor/github.com/mattbaird/elastigo
url = https://github.com/mattbaird/elastigo
[submodule "vendor/github.com/araddon/gou"]
path = vendor/github.com/araddon/gou
url = https://github.com/araddon/gou
[submodule "vendor/github.com/bitly/go-hostpool"]
path = vendor/github.com/bitly/go-hostpool
url = https://github.com/bitly/go-hostpool
elasticsearch.go
+52 -51
View File
@@ -2,8 +2,7 @@ package main
import ( import (
"encoding/json" "encoding/json"
"github.com/buger/elastigo/api" "github.com/mattbaird/elastigo/lib"
"github.com/buger/elastigo/core"
"github.com/buger/gor/proto" "github.com/buger/gor/proto"
"log" "log"
"regexp" "regexp"
@@ -19,33 +18,34 @@ func (e *ESUriErorr) Error() string {
type ESPlugin struct { type ESPlugin struct {
Active bool Active bool
ApiPort string ApiPort string
eConn *elastigo.Conn
Host string Host string
Index string Index string
indexor *core.BulkIndexer indexor *elastigo.BulkIndexer
done chan bool done chan bool
} }
type ESRequestResponse struct { type ESRequestResponse struct {
ReqUrl []byte `json:"Req_URL"` ReqURL string `json:"Req_URL"`
ReqMethod []byte `json:"Req_Method"` ReqMethod string `json:"Req_Method"`
ReqUserAgent []byte `json:"Req_User-Agent"` ReqUserAgent string `json:"Req_User-Agent"`
ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"` ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"`
ReqAccept []byte `json:"Req_Accept,omitempty"` ReqAccept string `json:"Req_Accept,omitempty"`
ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"` ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"`
ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"` ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"`
ReqConnection []byte `json:"Req_Connection,omitempty"` ReqConnection string `json:"Req_Connection,omitempty"`
ReqCookies []byte `json:"Req_Cookies,omitempty"` ReqCookies string `json:"Req_Cookies,omitempty"`
RespStatus []byte `json:"Resp_Status"` RespStatus string `json:"Resp_Status"`
RespStatusCode []byte `json:"Resp_Status-Code"` RespStatusCode string `json:"Resp_Status-Code"`
RespProto []byte `json:"Resp_Proto,omitempty"` RespProto string `json:"Resp_Proto,omitempty"`
RespContentLength []byte `json:"Resp_Content-Length,omitempty"` RespContentLength string `json:"Resp_Content-Length,omitempty"`
RespContentType []byte `json:"Resp_Content-Type,omitempty"` RespContentType string `json:"Resp_Content-Type,omitempty"`
RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"` RespTransferEncoding string `json:"Resp_Transfer-Encoding,omitempty"`
RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"` RespContentEncoding string `json:"Resp_Content-Encoding,omitempty"`
RespExpires []byte `json:"Resp_Expires,omitempty"` RespExpires string `json:"Resp_Expires,omitempty"`
RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"` RespCacheControl string `json:"Resp_Cache-Control,omitempty"`
RespVary []byte `json:"Resp_Vary,omitempty"` RespVary string `json:"Resp_Vary,omitempty"`
RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"` RespSetCookie string `json:"Resp_Set-Cookie,omitempty"`
Rtt int64 `json:"RTT"` Rtt int64 `json:"RTT"`
Timestamp time.Time Timestamp time.Time
} }
@@ -76,24 +76,24 @@ func (p *ESPlugin) Init(URI string) {
if err != nil { if err != nil {
log.Fatal("Can't initialize ElasticSearch plugin.", err) log.Fatal("Can't initialize ElasticSearch plugin.", err)
} }
p.eConn = elastigo.NewConn()
p.eConn.SetPort(p.ApiPort)
p.eConn.SetHosts([]string{p.Host})
api.Domain = p.Host p.indexor = p.eConn.NewBulkIndexerErrors(50, 60)
api.Port = p.ApiPort
p.indexor = core.NewBulkIndexerErrors(50, 60)
p.done = make(chan bool) p.done = make(chan bool)
p.indexor.Run(p.done) p.indexor.Start()
// Only start the ErrorHandler goroutine when in verbose mode // Only start the ErrorHandler goroutine when in verbose mode
// no need to burn ressources otherwise // no need to burn ressources otherwise
// go p.ErrorHandler() go p.ErrorHandler()
log.Println("Initialized Elasticsearch Plugin") log.Println("Initialized Elasticsearch Plugin")
return return
} }
func (p *ESPlugin) IndexerShutdown() { func (p *ESPlugin) IndexerShutdown() {
p.done <- true p.indexor.Stop()
return return
} }
@@ -118,28 +118,29 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
} }
t := time.Now() t := time.Now()
rtt := p.RttDurationToMs(stop.Sub(start)) rtt := p.RttDurationToMs(stop.Sub(start))
req = payloadBody(req)
esResp := ESRequestResponse{ esResp := ESRequestResponse{
ReqUrl: proto.Path(req), ReqURL: string(proto.Path(req)),
ReqMethod: proto.Method(req), ReqMethod: string(proto.Method(req)),
ReqUserAgent: proto.Header(req, []byte("User-Agent")), ReqUserAgent: string(proto.Header(req, []byte("User-Agent"))),
ReqAcceptLanguage: proto.Header(req, []byte("Accept-Language")), ReqAcceptLanguage: string(proto.Header(req, []byte("Accept-Language"))),
ReqAccept: proto.Header(req, []byte("Accept")), ReqAccept: string(proto.Header(req, []byte("Accept"))),
ReqAcceptEncoding: proto.Header(req, []byte("Accept-Encoding")), ReqAcceptEncoding: string(proto.Header(req, []byte("Accept-Encoding"))),
ReqIfModifiedSince: proto.Header(req, []byte("If-Modified-Since")), ReqIfModifiedSince: string(proto.Header(req, []byte("If-Modified-Since"))),
ReqConnection: proto.Header(req, []byte("Connection")), ReqConnection: string(proto.Header(req, []byte("Connection"))),
ReqCookies: proto.Header(req, []byte("Cookie")), ReqCookies: string(proto.Header(req, []byte("Cookie"))),
RespStatus: proto.Status(resp), RespStatus: string(proto.Status(resp)),
RespStatusCode: proto.Status(resp), RespStatusCode: string(proto.Status(resp)),
RespProto: proto.Method(resp), RespProto: string(proto.Method(resp)),
RespContentLength: proto.Header(resp, []byte("Content-Length")), RespContentLength: string(proto.Header(resp, []byte("Content-Length"))),
RespContentType: proto.Header(resp, []byte("Content-Type")), RespContentType: string(proto.Header(resp, []byte("Content-Type"))),
RespTransferEncoding: proto.Header(resp, []byte("Transfer-Encoding")), RespTransferEncoding: string(proto.Header(resp, []byte("Transfer-Encoding"))),
RespContentEncoding: proto.Header(resp, []byte("Content-Encoding")), RespContentEncoding: string(proto.Header(resp, []byte("Content-Encoding"))),
RespExpires: proto.Header(resp, []byte("Expires")), RespExpires: string(proto.Header(resp, []byte("Expires"))),
RespCacheControl: proto.Header(resp, []byte("Cache-Control")), RespCacheControl: string(proto.Header(resp, []byte("Cache-Control"))),
RespVary: proto.Header(resp, []byte("Vary")), RespVary: string(proto.Header(resp, []byte("Vary"))),
RespSetCookie: proto.Header(resp, []byte("Set-Cookie")), RespSetCookie: string(proto.Header(resp, []byte("Set-Cookie"))),
Rtt: rtt, Rtt: rtt,
Timestamp: t, Timestamp: t,
} }
@@ -147,7 +148,7 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} else { } else {
p.indexor.Index(p.Index, "RequestResponse", "", "", &t, j, true) p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
} }
return return
} }
vendor/github.com/araddon/gou
Generated Vendored Submodule
+1
Submodule vendor/github.com/araddon/gou added at 50a94aa4a3
vendor/github.com/bitly/go-hostpool
Generated Vendored Submodule
+1
Submodule vendor/github.com/bitly/go-hostpool added at d0e59c22a5
vendor/github.com/mattbaird/elastigo
Generated Vendored Submodule
+1
Submodule vendor/github.com/mattbaird/elastigo added at 34c4c4d842