Add support for VXLAN and VLAN (#1051)

Added support for capturing virtualized traffic.

## VXLAN
https://en.wikipedia.org/wiki/Virtual_Extensible_LAN

VXLAN implemented as separate engine, which opens UDP socket and awaits traffic. 
This approach is made to work with AWS Traffic Mirroring.

In order to enable VXLAN set `--input-raw-engine vxlan`

Example:
```
gor --input-raw :80 --input-raw-engine vxlan --output-stdout`
```

By default, it looks for vxlan traffic on the standard 4789 port, but you can override it with `--input-raw-vxlan-port`. 
Additionally, you can allow only specific VNIs using `--input-raw-vxlan-vni`, or disallow by using the same option, but by adding "minus" sign to the value: `--input-raw-vxlan-vni -2`. 

Example with all options:
```
gor --input-raw :80 --input-raw-engine vxlan --input-raw-vxlan-vni 1 --input-raw-vxlan-vni 2 --input-raw-vxlan-port 2222 --output-stdout
```

# VLAN
https://en.wikipedia.org/wiki/IEEE_802.1Q

VLAN protocol enabled using `--input-raw-vlan` argument, and you can filter for specific VLAN VIDs using `--input-raw-vlan-vid`. VLAN filtering happens on BPF level.

Example:
```
gor --input-raw :80 --input-raw-vlan --input-raw-vlan-vid 1 --output-stdout`
```

## Notes

Did a refactoring of RAW Input options, so it will be easy to extend in future.
This commit is contained in:
Leonid Bugaev
2022-02-12 10:10:26 +03:00
committed by GitHub
parent b520511c96
commit a5f5615156
7 changed files with 319 additions and 143 deletions
+103 -76
View File
@@ -41,34 +41,42 @@ type PcapStatProvider interface {
// PcapOptions options that can be set on a pcap capture handle,
// these options take effect on inactive pcap handles
type PcapOptions struct {
BufferTimeout time.Duration `json:"input-raw-buffer-timeout"`
TimestampType string `json:"input-raw-timestamp-type"`
BPFFilter string `json:"input-raw-bpf-filter"`
BufferSize size.Size `json:"input-raw-buffer-size"`
Promiscuous bool `json:"input-raw-promisc"`
Monitor bool `json:"input-raw-monitor"`
Snaplen bool `json:"input-raw-override-snaplen"`
BufferTimeout time.Duration `json:"input-raw-buffer-timeout"`
TimestampType string `json:"input-raw-timestamp-type"`
BPFFilter string `json:"input-raw-bpf-filter"`
BufferSize size.Size `json:"input-raw-buffer-size"`
Promiscuous bool `json:"input-raw-promisc"`
Monitor bool `json:"input-raw-monitor"`
Snaplen bool `json:"input-raw-override-snaplen"`
Engine EngineType `json:"input-raw-engine"`
VXLANPort int `json:"input-raw-vxlan-port"`
VXLANVNIs []int `json:"input-raw-vxlan-vni"`
VLAN bool `json:"input-raw-vlan"`
VLANVIDs []int `json:"input-raw-vlan-vid"`
Expire time.Duration `json:"input-raw-expire"`
TrackResponse bool `json:"input-raw-track-response"`
Protocol tcp.TCPProtocol `json:"input-raw-protocol"`
RealIPHeader string `json:"input-raw-realip-header"`
Stats bool `json:"input-raw-stats"`
AllowIncomplete bool `json:"input-raw-allow-incomplete"`
Transport string
}
// Listener handle traffic capture, this is its representation.
type Listener struct {
sync.Mutex
Transport string // transport layer default to tcp
config PcapOptions
Activate func() error // function is used to activate the engine. it must be called before reading packets
Handles map[string]packetHandle
Interfaces []pcap.Interface
loopIndex int
Reading chan bool // this channel is closed when the listener has started reading packets
PcapOptions
Engine EngineType
ports []uint16 // src or/and dst ports
trackResponse bool
expiry time.Duration
allowIncomplete bool
messages chan *tcp.Message
protocol tcp.TCPProtocol
messages chan *tcp.Message
host string // pcap file name or interface (name, hardware addr, index or ip address)
ports []uint16
host string // pcap file name or interface (name, hardware addr, index or ip address)
closeDone chan struct{}
quit chan struct{}
@@ -88,6 +96,7 @@ const (
EnginePcapFile
EngineRawSocket
EngineAFPacket
EngineVXLAN
)
// Set is here so that EngineType can implement flag.Var
@@ -101,6 +110,8 @@ func (eng *EngineType) Set(v string) error {
*eng = EngineRawSocket
case "af_packet":
*eng = EngineAFPacket
case "vxlan":
*eng = EngineVXLAN
default:
return fmt.Errorf("invalid engine %s", v)
}
@@ -117,6 +128,8 @@ func (eng *EngineType) String() (e string) {
e = "raw_socket"
case EngineAFPacket:
e = "af_packet"
case EngineVXLAN:
e = "vxlan"
default:
e = ""
}
@@ -126,7 +139,7 @@ func (eng *EngineType) String() (e string) {
// NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
// is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
// if there is an error it will be associated with getting network interfaces
func NewListener(host string, ports []uint16, transport string, engine EngineType, protocol tcp.TCPProtocol, trackResponse bool, expiry time.Duration, allowIncomplete bool) (l *Listener, err error) {
func NewListener(host string, ports []uint16, config PcapOptions) (l *Listener, err error) {
l = &Listener{}
l.host = host
@@ -135,34 +148,28 @@ func NewListener(host string, ports []uint16, transport string, engine EngineTyp
}
l.ports = ports
l.Transport = "tcp"
if transport != "" {
l.Transport = transport
}
l.config = config
l.config.Transport = "tcp"
l.Handles = make(map[string]packetHandle)
l.trackResponse = trackResponse
l.closeDone = make(chan struct{})
l.quit = make(chan struct{})
l.Reading = make(chan bool)
l.expiry = expiry
l.allowIncomplete = allowIncomplete
l.protocol = protocol
l.messages = make(chan *tcp.Message, 10000)
switch engine {
switch l.config.Engine {
default:
l.Engine = EnginePcap
l.Activate = l.activatePcap
case EngineRawSocket:
l.Engine = EngineRawSocket
l.Activate = l.activateRawSocket
case EngineAFPacket:
l.Engine = EngineAFPacket
l.Activate = l.activateAFPacket
case EnginePcapFile:
l.Engine = EnginePcapFile
l.Activate = l.activatePcapFile
return
case EngineVXLAN:
l.Activate = l.activateVxLanSocket
return
}
err = l.setInterfaces()
@@ -172,12 +179,6 @@ func NewListener(host string, ports []uint16, transport string, engine EngineTyp
return
}
// SetPcapOptions set pcap options for all yet to be actived pcap handles
// setting this on already activated handles will not have any effect
func (l *Listener) SetPcapOptions(opts PcapOptions) {
l.PcapOptions = opts
}
// Listen listens for packets from the handles, and call handler on every packet received
// until the context done signal is sent or there is unrecoverable error on all handles.
// this function must be called after activating pcap handles
@@ -216,18 +217,18 @@ func (l *Listener) Filter(ifi pcap.Interface) (filter string) {
hosts = interfaceAddresses(ifi)
}
filter = portsFilter(l.Transport, "dst", l.ports)
filter = portsFilter(l.config.Transport, "dst", l.ports)
if len(hosts) != 0 && !l.Promiscuous {
if len(hosts) != 0 && !l.config.Promiscuous {
filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts))
} else {
filter = fmt.Sprintf("(%s)", filter)
}
if l.trackResponse {
responseFilter := portsFilter(l.Transport, "src", l.ports)
if l.config.TrackResponse {
responseFilter := portsFilter(l.config.Transport, "src", l.ports)
if len(hosts) != 0 && !l.Promiscuous {
if len(hosts) != 0 && !l.config.Promiscuous {
responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts))
} else {
responseFilter = fmt.Sprintf("(%s)", responseFilter)
@@ -236,6 +237,16 @@ func (l *Listener) Filter(ifi pcap.Interface) (filter string) {
filter = fmt.Sprintf("%s or %s", filter, responseFilter)
}
if l.config.VLAN {
if len(l.config.VLANVIDs) > 0 {
for _, vi := range l.config.VLANVIDs {
filter = fmt.Sprintf("vlan %d and ", vi) + filter
}
} else {
filter = "vlan and " + filter
}
}
return
}
@@ -249,29 +260,29 @@ func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err erro
}
defer inactive.CleanUp()
if l.TimestampType != "" && l.TimestampType != "go" {
if l.config.TimestampType != "" && l.config.TimestampType != "go" {
var ts pcap.TimestampSource
ts, err = pcap.TimestampSourceFromString(l.TimestampType)
ts, err = pcap.TimestampSourceFromString(l.config.TimestampType)
fmt.Println("Setting custom Timestamp Source. Supported values: `go`, ", inactive.SupportedTimestamps())
err = inactive.SetTimestampSource(ts)
if err != nil {
return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
}
}
if l.Promiscuous {
if err = inactive.SetPromisc(l.Promiscuous); err != nil {
if l.config.Promiscuous {
if err = inactive.SetPromisc(l.config.Promiscuous); err != nil {
return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
}
}
if l.Monitor {
if err = inactive.SetRFMon(l.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
if l.config.Monitor {
if err = inactive.SetRFMon(l.config.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
}
}
var snap int
if !l.Snaplen {
if !l.config.Snaplen {
infs, _ := net.Interfaces()
for _, i := range infs {
if i.Name == ifi.Name {
@@ -288,16 +299,16 @@ func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err erro
if err != nil {
return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
}
if l.BufferSize > 0 {
err = inactive.SetBufferSize(int(l.BufferSize))
if l.config.BufferSize > 0 {
err = inactive.SetBufferSize(int(l.config.BufferSize))
if err != nil {
return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name)
}
}
if l.BufferTimeout == 0 {
l.BufferTimeout = 2000 * time.Millisecond
if l.config.BufferTimeout == 0 {
l.config.BufferTimeout = 2000 * time.Millisecond
}
err = inactive.SetTimeout(l.BufferTimeout)
err = inactive.SetTimeout(l.config.BufferTimeout)
if err != nil {
return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name)
}
@@ -306,7 +317,7 @@ func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err erro
return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
}
bpfFilter := l.BPFFilter
bpfFilter := l.config.BPFFilter
if bpfFilter == "" {
bpfFilter = l.Filter(ifi)
}
@@ -325,16 +336,16 @@ func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {
if err != nil {
return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
}
if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
if err = handle.SetPromiscuous(l.config.Promiscuous || l.config.Monitor); err != nil {
return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
}
if l.BPFFilter == "" {
l.BPFFilter = l.Filter(ifi)
if l.config.BPFFilter == "" {
l.config.BPFFilter = l.Filter(ifi)
}
fmt.Println("BPF Filter: ", l.BPFFilter)
if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
fmt.Println("BPF Filter: ", l.config.BPFFilter)
if err = handle.SetBPFFilter(l.config.BPFFilter); err != nil {
handle.Close()
return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.config.BPFFilter, ifi.Name)
}
handle.SetLoopbackIndex(int32(l.loopIndex))
return
@@ -357,7 +368,7 @@ func http1EndHint(m *tcp.Message) bool {
if m.MissingChunk() {
return false
}
req, res := http1StartHint(m.Packets()[0])
return proto.HasFullPayload(m, m.PacketData()...) && (req || res)
}
@@ -374,7 +385,7 @@ func (l *Listener) read() {
linkType := int(layers.LinkTypeEthernet)
if _, ok := hndl.handler.(*pcap.Handle); ok {
linkType = int(hndl.handler.(*pcap.Handle).LinkType())
linkSize, ok = pcapLinkTypeLength(linkType)
linkSize, ok = pcapLinkTypeLength(linkType, l.config.VLAN)
if !ok {
if os.Getenv("GORDEBUG") != "0" {
log.Printf("can not identify link type of an interface '%s'\n", key)
@@ -383,9 +394,9 @@ func (l *Listener) read() {
}
}
messageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.expiry, l.allowIncomplete)
messageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.config.Expire, l.config.AllowIncomplete)
if l.protocol == tcp.ProtocolHTTP {
if l.config.Protocol == tcp.ProtocolHTTP {
messageParser.Start = http1StartHint
messageParser.End = http1EndHint
}
@@ -408,7 +419,7 @@ func (l *Listener) read() {
default:
data, ci, err := hndl.handler.ReadPacketData()
if err == nil {
if l.TimestampType == "go" {
if l.config.TimestampType == "go" {
ci.Timestamp = time.Now()
}
@@ -483,6 +494,18 @@ func (l *Listener) activatePcap() error {
return nil
}
func (l *Listener) activateVxLanSocket() error {
handler, err := newVXLANHandler(l.config.VXLANPort, l.config.VXLANVNIs)
if err != nil {
return err
}
l.Handles["vxlan"] = packetHandle{
handler: handler,
}
return nil
}
func (l *Listener) activateRawSocket() error {
if runtime.GOOS != "linux" {
return fmt.Errorf("sock_raw is not stabilized on OS other than linux")
@@ -516,15 +539,15 @@ func (l *Listener) activatePcapFile() (err error) {
tmp := l.host
l.host = ""
l.BPFFilter = l.Filter(pcap.Interface{})
l.config.BPFFilter = l.Filter(pcap.Interface{})
l.host = tmp
if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
if e = handle.SetBPFFilter(l.config.BPFFilter); e != nil {
handle.Close()
return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.config.BPFFilter)
}
fmt.Println("BPF Filter:", l.BPFFilter)
fmt.Println("BPF Filter:", l.config.BPFFilter)
l.Handles["pcap_file"] = packetHandle{
handler: handle,
@@ -547,11 +570,11 @@ func (l *Listener) activateAFPacket() error {
continue
}
if l.BPFFilter == "" {
l.BPFFilter = l.Filter(ifi)
if l.config.BPFFilter == "" {
l.config.BPFFilter = l.Filter(ifi)
}
fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.BPFFilter)
handle.SetBPFFilter(l.BPFFilter, 64<<10)
fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.config.BPFFilter)
handle.SetBPFFilter(l.config.BPFFilter, 64<<10)
l.Handles[ifi.Name] = packetHandle{
handler: handle,
@@ -681,10 +704,14 @@ func hostsFilter(direction string, hosts []string) string {
return strings.Join(hostsFilters, " or ")
}
func pcapLinkTypeLength(lType int) (int, bool) {
func pcapLinkTypeLength(lType int, vlan bool) (int, bool) {
switch layers.LinkType(lType) {
case layers.LinkTypeEthernet:
return 14, true
if vlan {
return 18, true
} else {
return 14, true
}
case layers.LinkTypeNull, layers.LinkTypeLoop:
return 4, true
case layers.LinkTypeRaw, 12, 14:
+100
View File
@@ -0,0 +1,100 @@
package capture
import (
"errors"
"fmt"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
"net"
"time"
)
const VxLanPacketSize = 1526 //vxlan 8 B + ethernet II 1518 B
type vxlanHandle struct {
connection *net.UDPConn
packetChannel chan gopacket.Packet
vnis []int
}
func newVXLANHandler(port int, vnis []int) (*vxlanHandle, error) {
if port == 0 {
port = 4789
}
addr := net.UDPAddr{
Port: port,
IP: net.ParseIP("0.0.0.0"),
}
vxlanHandle := &vxlanHandle{}
con, err := net.ListenUDP("udp", &addr)
if err != nil {
return nil, fmt.Errorf(err.Error())
}
vxlanHandle.connection = con
vxlanHandle.packetChannel = make(chan gopacket.Packet, 1000)
vxlanHandle.vnis = vnis
go vxlanHandle.reader()
return vxlanHandle, nil
}
func (v *vxlanHandle) reader() {
for {
inputBytes := make([]byte, VxLanPacketSize)
length, _, err := v.connection.ReadFromUDP(inputBytes)
if err != nil {
if errors.Is(err, net.ErrClosed) {
return
}
continue
}
packet := gopacket.NewPacket(inputBytes[:length], layers.LayerTypeVXLAN, gopacket.NoCopy)
ci := packet.Metadata()
ci.Timestamp = time.Now()
ci.CaptureLength = length
ci.Length = length
if len(v.vnis) > 0 && !v.vniIsAllowed(packet) {
continue
}
v.packetChannel <- packet
}
}
func (v *vxlanHandle) vniIsAllowed(packet gopacket.Packet) bool {
defaultState := false
if layer := packet.Layer(layers.LayerTypeVXLAN); layer != nil {
vxlan, _ := layer.(*layers.VXLAN)
for _, vn := range v.vnis {
if vn > 0 && int(vxlan.VNI) == vn {
return true
}
if vn < 0 {
if int(vxlan.VNI) == -vn {
return false
}
defaultState = true
}
}
}
return defaultState
}
func (v *vxlanHandle) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) {
packet := <-v.packetChannel
layer := packet.Layer(layers.LayerTypeVXLAN)
bytes := layer.LayerPayload()
return bytes, packet.Metadata().CaptureInfo, nil
}
func (v *vxlanHandle) Close() error {
if v.connection != nil {
return v.connection.Close()
}
return nil
}
+5
View File
@@ -37,6 +37,11 @@ sudo gor --input-raw :80 --input-raw-engine "raw_socket" --output-http "http://s
You can read more about [[Replaying HTTP traffic]].
You can use VXLAN or traffic mirroring from AWS to capture the traffic. The 4789 UDP port will be opened and that works as you are launched GoReplay on the source machine.
```
gor --input-raw :80 --input-raw-engine vxlan -output-stdout
```
### Tracking original IP addresses
You can use `--input-raw-realip-header` option to specify header name: If not blank, injects header with given name and real IP value to the request payload. Usually, this header should be named: `X-Real-IP`, but you can specify any name.
+12 -21
View File
@@ -8,45 +8,36 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/buger/goreplay/capture"
"github.com/buger/goreplay/proto"
"github.com/buger/goreplay/size"
"github.com/buger/goreplay/tcp"
)
// RAWInputConfig represents configuration that can be applied on raw input
type RAWInputConfig struct {
capture.PcapOptions
Expire time.Duration `json:"input-raw-expire"`
CopyBufferSize size.Size `json:"copy-buffer-size"`
Engine capture.EngineType `json:"input-raw-engine"`
TrackResponse bool `json:"input-raw-track-response"`
Protocol tcp.TCPProtocol `json:"input-raw-protocol"`
RealIPHeader string `json:"input-raw-realip-header"`
Stats bool `json:"input-raw-stats"`
AllowIncomplete bool `json:"input-raw-allow-incomplete"`
quit chan bool // Channel used only to indicate goroutine should shutdown
host string
ports []uint16
}
// RAWInput used for intercepting traffic for given address
type RAWInput struct {
sync.Mutex
RAWInputConfig
config RAWInputConfig
messageStats []tcp.Stats
listener *capture.Listener
messageParser *tcp.MessageParser
cancelListener context.CancelFunc
closed bool
quit chan bool // Channel used only to indicate goroutine should shutdown
host string
ports []uint16
}
// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
i = new(RAWInput)
i.RAWInputConfig = config
i.config = config
i.quit = make(chan bool)
host, _ports, err := net.SplitHostPort(address)
@@ -62,7 +53,7 @@ func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
}
if strings.HasSuffix(host, "pcap") {
i.RAWInputConfig.Engine = capture.EnginePcapFile
i.config.Engine = capture.EnginePcapFile
}
var ports []uint16
@@ -101,8 +92,8 @@ func (i *RAWInput) PluginRead() (*Message, error) {
var msgType byte = ResponsePayload
if msgTCP.Direction == tcp.DirIncoming {
msgType = RequestPayload
if i.RealIPHeader != "" {
msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
if i.config.RealIPHeader != "" {
msg.Data = proto.SetHeader(msg.Data, []byte(i.config.RealIPHeader), []byte(msgTCP.SrcAddr))
}
}
msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
@@ -115,7 +106,7 @@ func (i *RAWInput) PluginRead() (*Message, error) {
if msgTCP.TimedOut {
Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
}
if i.Stats {
if i.config.Stats {
stat := msgTCP.Stats
go i.addStats(stat)
}
@@ -125,11 +116,11 @@ func (i *RAWInput) PluginRead() (*Message, error) {
func (i *RAWInput) listen(address string) {
var err error
i.listener, err = capture.NewListener(i.host, i.ports, "", i.Engine, i.Protocol, i.TrackResponse, i.Expire, i.AllowIncomplete)
i.listener, err = capture.NewListener(i.host, i.ports, i.config.PcapOptions)
if err != nil {
log.Fatal(err)
}
i.listener.SetPcapOptions(i.PcapOptions)
err = i.listener.Activate()
if err != nil {
log.Fatal(err)
+1 -1
View File
@@ -106,7 +106,7 @@ func NewPlugins() *InOutPlugins {
}
for _, options := range Settings.InputRAW {
plugins.registerPlugin(NewRAWInput, options, Settings.RAWInputConfig)
plugins.registerPlugin(NewRAWInput, options, Settings.InputRAWConfig)
}
for _, options := range Settings.InputTCP {
+81 -38
View File
@@ -4,23 +4,59 @@ import (
"flag"
"fmt"
"os"
"strconv"
"sync"
"time"
"github.com/buger/goreplay/size"
)
// DEMO indicates that goreplay is running in demo mode
var DEMO string
// MultiOption allows to specify multiple flags with same name and collects all values into array
type MultiOption []string
type MultiOption struct {
a *[]string
}
func (h *MultiOption) String() string {
return fmt.Sprint(*h)
if h.a == nil {
return ""
}
return fmt.Sprint(*h.a)
}
// Set gets called multiple times for each flag with same name
func (h *MultiOption) Set(value string) error {
*h = append(*h, value)
if h.a == nil {
return nil
}
*h.a = append(*h.a, value)
return nil
}
// MultiOption allows to specify multiple flags with same name and collects all values into array
type MultiIntOption struct {
a *[]int
}
func (h *MultiIntOption) String() string {
if h.a == nil {
return ""
}
return fmt.Sprint(*h.a)
}
// Set gets called multiple times for each flag with same name
func (h *MultiIntOption) Set(value string) error {
if h.a == nil {
return nil
}
val, _ := strconv.Atoi(value)
*h.a = append(*h.a, val)
return nil
}
@@ -34,37 +70,39 @@ type AppSettings struct {
RecognizeTCPSessions bool `json:"recognize-tcp-sessions"`
Pprof string `json:"http-pprof"`
InputDummy MultiOption `json:"input-dummy"`
OutputDummy MultiOption
CopyBufferSize size.Size `json:"copy-buffer-size"`
InputDummy []string `json:"input-dummy"`
OutputDummy []string
OutputStdout bool `json:"output-stdout"`
OutputNull bool `json:"output-null"`
InputTCP MultiOption `json:"input-tcp"`
InputTCP []string `json:"input-tcp"`
InputTCPConfig TCPInputConfig
OutputTCP MultiOption `json:"output-tcp"`
OutputTCP []string `json:"output-tcp"`
OutputTCPConfig TCPOutputConfig
OutputTCPStats bool `json:"output-tcp-stats"`
InputFile MultiOption `json:"input-file"`
InputFile []string `json:"input-file"`
InputFileLoop bool `json:"input-file-loop"`
InputFileReadDepth int `json:"input-file-read-depth"`
InputFileDryRun bool `json:"input-file-dry-run"`
InputFileMaxWait time.Duration `json:"input-file-max-wait"`
OutputFile MultiOption `json:"output-file"`
OutputFile []string `json:"output-file"`
OutputFileConfig FileOutputConfig
InputRAW MultiOption `json:"input_raw"`
RAWInputConfig
InputRAW []string `json:"input_raw"`
InputRAWConfig RAWInputConfig
Middleware string `json:"middleware"`
InputHTTP MultiOption
OutputHTTP MultiOption `json:"output-http"`
PrettifyHTTP bool `json:"prettify-http"`
InputHTTP []string
OutputHTTP []string `json:"output-http"`
PrettifyHTTP bool `json:"prettify-http"`
OutputHTTPConfig HTTPOutputConfig
OutputBinary MultiOption `json:"output-binary"`
OutputBinary []string `json:"output-binary"`
OutputBinaryConfig BinaryOutputConfig
ModifierConfig HTTPModifierConfig
@@ -98,29 +136,29 @@ func init() {
flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
flag.Var(&MultiOption{&Settings.InputDummy}, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
flag.Var(&MultiOption{&Settings.InputTCP}, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
flag.BoolVar(&Settings.InputTCPConfig.Secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.")
flag.StringVar(&Settings.InputTCPConfig.CertificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.")
flag.StringVar(&Settings.InputTCPConfig.KeyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.")
flag.Var(&Settings.OutputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
flag.Var(&MultiOption{&Settings.OutputTCP}, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
flag.BoolVar(&Settings.OutputTCPConfig.Secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.")
flag.BoolVar(&Settings.OutputTCPConfig.SkipVerify, "output-tcp-skip-verify", false, "Don't verify hostname on TLS secure connection.")
flag.BoolVar(&Settings.OutputTCPConfig.Sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.")
flag.IntVar(&Settings.OutputTCPConfig.Workers, "output-tcp-workers", 10, "Number of parallel tcp connections, default is 10")
flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
flag.Var(&MultiOption{&Settings.InputFile}, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
flag.IntVar(&Settings.InputFileReadDepth, "input-file-read-depth", 100, "GoReplay tries to read and cache multiple records, in advance. In parallel it also perform sorting of requests, if they came out of order. Since it needs hold this buffer in memory, bigger values can cause worse performance")
flag.BoolVar(&Settings.InputFileDryRun, "input-file-dry-run", false, "Simulate reading from the data source without replaying it. You will get information about expected replay time, number of found records etc.")
flag.DurationVar(&Settings.InputFileMaxWait, "input-file-max-wait", 0, "Set the maximum time between requests. Can help in situations when you have too long periods between request, and you want to skip them. Example: --input-raw-max-wait 1s")
flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
flag.Var(&MultiOption{&Settings.OutputFile}, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
flag.BoolVar(&Settings.OutputFileConfig.Append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
flag.Var(&Settings.OutputFileConfig.SizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
@@ -131,27 +169,32 @@ func init() {
flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout")
// input raw flags
flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
flag.BoolVar(&Settings.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
flag.Var(&Settings.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket` or `pcap_file`")
flag.Var(&Settings.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
flag.StringVar(&Settings.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
flag.DurationVar(&Settings.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
flag.StringVar(&Settings.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
flag.StringVar(&Settings.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)")
flag.BoolVar(&Settings.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
flag.DurationVar(&Settings.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
flag.Var(&Settings.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
flag.BoolVar(&Settings.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
flag.BoolVar(&Settings.AllowIncomplete, "input-raw-allow-incomplete", false, "If turned on Gor will record HTTP messages with missing packets")
// input raw flags
flag.Var(&MultiOption{&Settings.InputRAW}, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
flag.BoolVar(&Settings.InputRAWConfig.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
flag.IntVar(&Settings.InputRAWConfig.VXLANPort, "input-raw-vxlan-port", 4789, "VXLAN port. Can be used only when engine set to `vxlan`. Default: 4789")
flag.Var(&MultiIntOption{&Settings.InputRAWConfig.VXLANVNIs}, "input-raw-vxlan-vni", "VXLAN VNI to capture. By default capture all VNIs. Ignore VNI by setting them with minus sign, example: `--input-raw-vxlan-vni -2`")
flag.BoolVar(&Settings.InputRAWConfig.VLAN, "input-raw-vlan", false, "Enable VLAN (802.1Q) support")
flag.Var(&MultiIntOption{&Settings.InputRAWConfig.VLANVIDs}, "input-raw-vlan-vid", "VLAN VID to capture. By default capture all VIDs")
flag.Var(&Settings.InputRAWConfig.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket`, `pcap_file`, `vxlan`")
flag.Var(&Settings.InputRAWConfig.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
flag.StringVar(&Settings.InputRAWConfig.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
flag.DurationVar(&Settings.InputRAWConfig.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
flag.StringVar(&Settings.InputRAWConfig.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
flag.StringVar(&Settings.InputRAWConfig.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
flag.BoolVar(&Settings.InputRAWConfig.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
flag.DurationVar(&Settings.InputRAWConfig.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
flag.Var(&Settings.InputRAWConfig.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
flag.BoolVar(&Settings.InputRAWConfig.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
flag.BoolVar(&Settings.InputRAWConfig.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
flag.BoolVar(&Settings.InputRAWConfig.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
flag.BoolVar(&Settings.InputRAWConfig.AllowIncomplete, "input-raw-allow-incomplete", false, "If turned on Gor will record HTTP messages with missing packets")
flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
flag.Var(&Settings.OutputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
flag.Var(&MultiOption{&Settings.OutputHTTP}, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
/* outputHTTPConfig */
flag.Var(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", "HTTP response buffer size, all data after this size will be discarded.")
@@ -171,7 +214,7 @@ func init() {
flag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
/* outputHTTPConfig */
flag.Var(&Settings.OutputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
flag.Var(&MultiOption{&Settings.OutputBinary}, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
/* outputBinaryConfig */
flag.Var(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", "TCP response buffer size, all data after this size will be discarded.")
+17 -7
View File
@@ -289,13 +289,11 @@ func (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet {
}
for _, p := range parser.ports {
if pckt.DstPort == p {
for _, ip := range parser.ips {
if pckt.DstIP.Equal(ip) {
pckt.Direction = DirIncoming
break
}
}
if pckt.DstPort == p && containsOrEmpty(pckt.DstIP, parser.ips) {
pckt.Direction = DirIncoming
break
} else if pckt.SrcPort == p && containsOrEmpty(pckt.SrcIP, parser.ips) {
pckt.Direction = DirOutcoming
break
}
}
@@ -303,6 +301,18 @@ func (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet {
return pckt
}
func containsOrEmpty(element net.IP, ipList []net.IP) bool {
if len(ipList) == 0 {
return true
}
for _, ip := range ipList {
if ip.Equal(element) {
return true
}
}
return false
}
func (parser *MessageParser) processPacket(pckt *Packet) {
if pckt == nil {
return