Improving docs

README.md

@@ -61,7 +61,7 @@ gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http:/
 ```
 
 ### HTTP output workers
-By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10.  The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue.  No more workers will be spawned until that request to spawn N workers is satisfied.  If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies.  
+By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10.  The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue.  No more workers will be spawned until that request to spawn N workers is satisfied.  If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies.
 You may specify fixed number of workers using  `--output-http-workers=20` option.
 
 ### Follow redirects
@@ -149,7 +149,7 @@ gor --input-raw :80 --output-http "http://staging.server" \
 ```
 
 ### Rewriting original request
-Gor supports built-in basic rewriting support, for complex logic see https://github.com/buger/gor/pull/162
+Gor supports some basic request rewriting support. For complex logic you can use middleware, see below.
 
 #### Rewrite URL based on a mapping
 ```
@@ -177,6 +177,25 @@ Host header gets special treatment. By default Host get set to the value specifi
 
 If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all.
 
+### Middleware
+Middleware is a programm that accepts request payload and response at STDIN and emits modified requests at STDOUT.
+
+```
+                   Original request      +--------------+
++-------------+----------STDIN---------->+              |
+|  Gor input  |                          |  Middleware  |
++-------------+----------STDIN---------->+              |
+                   Original response     +------+---+---+
+                                                |   ^
++-------------+    Modified request             v   |
+| Gor output  +<---------STDOUT-----------------+   |
++-----+-------+                                     |
+      |                                             |
+      |            Replayed response                |
+      +------------------STDIN----------------->----+
+```
+
+
 ### Saving requests to file and replaying them
 You can save requests to file, and replay them later:
 ```

examples/echo_modifier.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-while read line; do
-    decoded=$(echo -e "$line" | xxd -r -p)
-
-    header=$(echo -e "$decoded" | head -n +1)
-    payload=$(echo -e "$decoded" | tail -n +2)
-
-    encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n")
-
-    >&2 echo ""
-    >&2 echo "[DEBUG][ECHO] ==================================="
-
-    case ${header:0:1} in
-    "2")
-        >&2 echo "[DEBUG][ECHO] Request type: Original Response"
-        ;;
-    "3")
-        >&2 echo "[DEBUG][ECHO] Request type: Replayed Response"
-        ;;
-    "1")
-        >&2 echo "[DEBUG][ECHO] Request type: Request"
-        echo "$encoded"
-        ;;
-    *)
-        >&2 echo "[DEBUG][ECHO] Unknown request type $header"
-    esac
-    >&2 echo "[DEBUG][ECHO] ==================================="
-
-    >&2 echo "[DEBUG][ECHO] Original data: $line"
-    >&2 echo "[DEBUG][ECHO] Decoded request: $decoded"
-    >&2 echo "[DEBUG][ECHO] Encoded data: $encoded"
-done;

examples/middleware/echo_modifier.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# `xxd` utility included into vim-common package
+# It allow hex decoding/encoding
+
+function log {
+    # Logging to stderr, because stdout/stdin used for data transfer
+    >&2 echo "[DEBUG][ECHO] $1"
+}
+
+while read line; do
+    decoded=$(echo -e "$line" | xxd -r -p)
+
+    header=$(echo -e "$decoded" | head -n +1)
+    payload=$(echo -e "$decoded" | tail -n +2)
+
+    encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n")
+
+    log ""
+    log "==================================="
+
+    case ${header:0:1} in
+    "1")
+        log "Request type: Request"
+        echo "$encoded"
+        ;;
+    "2")
+        log "Request type: Original Response"
+        ;;
+    "3")
+        log "Request type: Replayed Response"
+        ;;
+    *)
+        log "Unknown request type $header"
+    esac
+    log "==================================="
+
+    log "Original data: $line"
+    log "Decoded request: $decoded"
+    log "Encoded data: $encoded"
+done;

examples/middleware/token_modifier.go

@@ -45,7 +45,7 @@ func process(buf []byte) {
 	Debug("Received payload:", string(buf))
 
 	switch payloadType {
-	case '1':
+	case '1': // Request
 		if bytes.Equal(proto.Path(payload), []byte("/token")) {
 			originalTokens[reqID] = []byte{}
 			Debug("Found token request:", reqID)

input_raw_test.go

@@ -244,6 +244,4 @@ func TestInputRAWResponse(t *testing.T) {
 
 	wg.Wait()
 	close(quit)
-
-	time.Sleep(100 * time.Millisecond)
 }

middleware_test.go

@@ -108,7 +108,7 @@ func TestEchoMiddleware(t *testing.T) {
 
 	quit := make(chan int)
 
-	Settings.middleware = "./examples/echo_modifier.sh"
+	Settings.middleware = "./examples/middleware/echo_modifier.sh"
 
 	// Catch traffic from one service
 	input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true)
@@ -173,12 +173,13 @@ func TestTokenMiddleware(t *testing.T) {
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}
-	Settings.middleware = "go run ./examples/token_modifier.go"
+	Settings.middleware = "go run ./examples/middleware/token_modifier.go"
 
 	// Start Gor
 	go Start(quit)
 
 	// Wait for middleware to initialize
+	// Give go compiller time to build programm
 	time.Sleep(500 * time.Millisecond)
 
 	// Should receive 2 requests from original + 2 from replayed