From a18079fa840112678ac1e3a1269871aa7bdfbdbf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 10:33:50 +0300 Subject: [PATCH] Improving docs --- README.md | 23 +++++++++++- examples/echo_modifier.sh | 32 ---------------- examples/{ => middleware}/echo_modifier.rb | 0 examples/middleware/echo_modifier.sh | 41 +++++++++++++++++++++ examples/{ => middleware}/token_modifier.go | 2 +- input_raw_test.go | 2 - middleware_test.go | 5 ++- 7 files changed, 66 insertions(+), 39 deletions(-) delete mode 100755 examples/echo_modifier.sh rename examples/{ => middleware}/echo_modifier.rb (100%) create mode 100755 examples/middleware/echo_modifier.sh rename examples/{ => middleware}/token_modifier.go (99%) diff --git a/README.md b/README.md index 2ea19c3..67f02e6 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ gor --input-tcp :28020 --output-http "http://staging.com" --output-http "http:/ ``` ### HTTP output workers -By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. +By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. You may specify fixed number of workers using `--output-http-workers=20` option. ### Follow redirects @@ -149,7 +149,7 @@ gor --input-raw :80 --output-http "http://staging.server" \ ``` ### Rewriting original request -Gor supports built-in basic rewriting support, for complex logic see https://github.com/buger/gor/pull/162 +Gor supports some basic request rewriting support. For complex logic you can use middleware, see below. #### Rewrite URL based on a mapping ``` @@ -177,6 +177,25 @@ Host header gets special treatment. By default Host get set to the value specifi If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. +### Middleware +Middleware is a programm that accepts request payload and response at STDIN and emits modified requests at STDOUT. + +``` + Original request +--------------+ ++-------------+----------STDIN---------->+ | +| Gor input | | Middleware | ++-------------+----------STDIN---------->+ | + Original response +------+---+---+ + | ^ ++-------------+ Modified request v | +| Gor output +<---------STDOUT-----------------+ | ++-----+-------+ | + | | + | Replayed response | + +------------------STDIN----------------->----+ +``` + + ### Saving requests to file and replaying them You can save requests to file, and replay them later: ``` diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh deleted file mode 100755 index 13d2074..0000000 --- a/examples/echo_modifier.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -while read line; do - decoded=$(echo -e "$line" | xxd -r -p) - - header=$(echo -e "$decoded" | head -n +1) - payload=$(echo -e "$decoded" | tail -n +2) - - encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") - - >&2 echo "" - >&2 echo "[DEBUG][ECHO] ===================================" - - case ${header:0:1} in - "2") - >&2 echo "[DEBUG][ECHO] Request type: Original Response" - ;; - "3") - >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" - ;; - "1") - >&2 echo "[DEBUG][ECHO] Request type: Request" - echo "$encoded" - ;; - *) - >&2 echo "[DEBUG][ECHO] Unknown request type $header" - esac - >&2 echo "[DEBUG][ECHO] ===================================" - - >&2 echo "[DEBUG][ECHO] Original data: $line" - >&2 echo "[DEBUG][ECHO] Decoded request: $decoded" - >&2 echo "[DEBUG][ECHO] Encoded data: $encoded" -done; diff --git a/examples/echo_modifier.rb b/examples/middleware/echo_modifier.rb similarity index 100% rename from examples/echo_modifier.rb rename to examples/middleware/echo_modifier.rb diff --git a/examples/middleware/echo_modifier.sh b/examples/middleware/echo_modifier.sh new file mode 100755 index 0000000..96dc121 --- /dev/null +++ b/examples/middleware/echo_modifier.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# `xxd` utility included into vim-common package +# It allow hex decoding/encoding + +function log { + # Logging to stderr, because stdout/stdin used for data transfer + >&2 echo "[DEBUG][ECHO] $1" +} + +while read line; do + decoded=$(echo -e "$line" | xxd -r -p) + + header=$(echo -e "$decoded" | head -n +1) + payload=$(echo -e "$decoded" | tail -n +2) + + encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") + + log "" + log "===================================" + + case ${header:0:1} in + "1") + log "Request type: Request" + echo "$encoded" + ;; + "2") + log "Request type: Original Response" + ;; + "3") + log "Request type: Replayed Response" + ;; + *) + log "Unknown request type $header" + esac + log "===================================" + + log "Original data: $line" + log "Decoded request: $decoded" + log "Encoded data: $encoded" +done; diff --git a/examples/token_modifier.go b/examples/middleware/token_modifier.go similarity index 99% rename from examples/token_modifier.go rename to examples/middleware/token_modifier.go index 89ec995..1245c5f 100644 --- a/examples/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -45,7 +45,7 @@ func process(buf []byte) { Debug("Received payload:", string(buf)) switch payloadType { - case '1': + case '1': // Request if bytes.Equal(proto.Path(payload), []byte("/token")) { originalTokens[reqID] = []byte{} Debug("Found token request:", reqID) diff --git a/input_raw_test.go b/input_raw_test.go index 3276e7a..3273035 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -244,6 +244,4 @@ func TestInputRAWResponse(t *testing.T) { wg.Wait() close(quit) - - time.Sleep(100 * time.Millisecond) } diff --git a/middleware_test.go b/middleware_test.go index 6dd5b29..25d3374 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -108,7 +108,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) - Settings.middleware = "./examples/echo_modifier.sh" + Settings.middleware = "./examples/middleware/echo_modifier.sh" // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) @@ -173,12 +173,13 @@ func TestTokenMiddleware(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "go run ./examples/token_modifier.go" + Settings.middleware = "go run ./examples/middleware/token_modifier.go" // Start Gor go Start(quit) // Wait for middleware to initialize + // Give go compiller time to build programm time.Sleep(500 * time.Millisecond) // Should receive 2 requests from original + 2 from replayed