From eede2aacf64154bf89eacac31a8a5f0cc1e4d968 Mon Sep 17 00:00:00 2001
From: Allan-QLB <68638598+Allan-QLB@users.noreply.github.com>
Date: Wed, 20 Sep 2023 15:56:00 +0800
Subject: [PATCH] Improve token parsing and better support control operators
 such as pipe and redirections (#2658)

---
 .../core/shell/cli/impl/CliTokenImpl.java     |  73 +--
 .../core/shell/cli/impl/CliTokenImplTest.java | 420 ++++++++++++++++--
 2 files changed, 435 insertions(+), 58 deletions(-)

diff --git a/core/src/main/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImpl.java b/core/src/main/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImpl.java
index ca956e556..175f4cd82 100644
--- a/core/src/main/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImpl.java
+++ b/core/src/main/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImpl.java
@@ -11,6 +11,9 @@ import java.util.List;
  * @author <a href="mailto:julien@julienviet.com">Julien Viet</a>
  */
 public class CliTokenImpl implements CliToken {
+    private static final String PIPE = "|";
+    private static final String REDIRECT = ">";
+    private static final String REDIRECT_APPEND = ">>";
 
     final boolean text;
     final String raw;
@@ -71,7 +74,7 @@ public class CliTokenImpl implements CliToken {
 
         tokenize(s, 0, tokens);
 
-        tokens = correctPipeChar(tokens);
+        tokens = adjustTokensForSpecialSymbols(tokens);
         return tokens;
 
     }
@@ -87,35 +90,53 @@ public class CliTokenImpl implements CliToken {
      * unsupported:
      * 3) thread|grep xxx
      * 4) trace -E  classA|classB methodA|methodB|grep classA
-     * @param tokens
-     * @return
+     *
+     * Also handles redirection of '>' and '>>' in the similar way
+     *
+     * @param tokens original tokens
+     * @return adjusted tokens
      */
-    private static List<CliToken> correctPipeChar(List<CliToken> tokens) {
-        List<CliToken> newTokens = new ArrayList<CliToken>(tokens.size()+4);
-        for (CliToken token : tokens) {
-            String tokenValue = token.value();
-            if (tokenValue.length()>1 && tokenValue.endsWith("|")) {
-                //split last char '|'
-                tokenValue = tokenValue.substring(0, tokenValue.length()-1);
-                String rawValue = token.raw();
-                rawValue = rawValue.substring(0, rawValue.length()-1);
-                newTokens.add(new CliTokenImpl(token.isText(), rawValue, tokenValue));
-                //add '|' char
-                newTokens.add(new CliTokenImpl(true, "|", "|"));
+    private static List<CliToken> adjustTokensForSpecialSymbols(List<CliToken> tokens) {
+        return separateLeadingAndTailingSymbol(tokens, PIPE, REDIRECT_APPEND, REDIRECT);
+    }
 
-            } else if (tokenValue.length()>1 && tokenValue.startsWith("|")) {
-                //add '|' char
-                newTokens.add(new CliTokenImpl(true, "|", "|"));
-                //remove first char '|'
-                tokenValue = tokenValue.substring(1);
-                String rawValue = token.raw();
-                rawValue = rawValue.substring(1);
-                newTokens.add(new CliTokenImpl(token.isText(), rawValue, tokenValue));
-            } else {
-                newTokens.add(token);
+    private static List<CliToken> separateLeadingAndTailingSymbol(List<CliToken> tokens, String... symbols) {
+        List<CliToken> adjustedTokens = new ArrayList<>();
+        for (CliToken token : tokens) {
+            String value = token.value();
+            String raw = token.raw();
+            boolean handled = false;
+            for (String symbol : symbols) {
+                if (value.equals(symbol)) {
+                    break;
+                } else if (value.endsWith(symbol)) {
+                    handled = true;
+                    int lastIndexOfSymbol = raw.lastIndexOf(symbol);
+                    adjustedTokens.add(new CliTokenImpl(
+                            token.isText(),
+                            raw.substring(0, lastIndexOfSymbol),
+                            value.substring(0, value.length() - symbol.length())
+                    ));
+                    adjustedTokens.add(new CliTokenImpl(true, raw.substring(lastIndexOfSymbol), symbol));
+                    break;
+                } else if (value.startsWith(symbol)) {
+                    handled = true;
+                    int firstIndexOfSymbol = raw.indexOf(symbol);
+                    adjustedTokens.add(new CliTokenImpl(true,
+                            raw.substring(0, firstIndexOfSymbol + symbol.length()), symbol));
+                    adjustedTokens.add(new CliTokenImpl(
+                            token.isText(),
+                            raw.substring(firstIndexOfSymbol + symbol.length()),
+                            value.substring(symbol.length())
+                    ));
+                    break;
+                }
+            }
+            if (!handled) {
+                adjustedTokens.add(token);
             }
         }
-        return newTokens;
+        return adjustedTokens;
     }
 
     private static void tokenize(String s, int index, List<CliToken> builder) {
diff --git a/core/src/test/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImplTest.java b/core/src/test/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImplTest.java
index fbaeb1b23..8e347d3ad 100644
--- a/core/src/test/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImplTest.java
+++ b/core/src/test/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImplTest.java
@@ -4,7 +4,7 @@ import com.taobao.arthas.core.shell.cli.CliToken;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.util.Iterator;
+import java.util.ArrayList;
 import java.util.List;
 
 public class CliTokenImplTest {
@@ -25,14 +25,86 @@ public class CliTokenImplTest {
     @Test
     public void testSupportedPipeCharWithoutRegex() {
         String[] expectedTextTokenValue = new String[]{"thread", "|", "grep", "xxx"};
-        List<CliToken> actualTokens = CliTokenImpl.tokenize("thread| grep xxx");
-        assertEquals(expectedTextTokenValue, actualTokens);
+        String cmd = "thread| grep xxx";
+        List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread | grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread |grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread'|' grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread '|' grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread '|'grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread\"|\" grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread \"|\" grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread \"|\"grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"thread| grep", "xxx"};
+        cmd = "thread'| 'grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread\"| \"grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"thread |grep", "xxx"};
+        cmd = "thread' |'grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "thread\" |\"grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"thread \"|\"grep", "xxx"};
+        cmd = "thread' \"|\"'grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"thread '|'grep", "xxx"};
+        cmd = "thread\" '|'\"grep xxx";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
 
-        actualTokens = CliTokenImpl.tokenize("thread | grep xxx");
-        assertEquals(expectedTextTokenValue, actualTokens);
 
-        actualTokens = CliTokenImpl.tokenize("thread |grep xxx");
-        assertEquals(expectedTextTokenValue, actualTokens);
     }
 
     /**
@@ -51,14 +123,85 @@ public class CliTokenImplTest {
     @Test
     public void testSupportedPipeCharWithRegex() {
         String[] expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB", "|", "grep", "classA"};
-        List<CliToken> actualTokens = CliTokenImpl.tokenize("trace -E classA|classB methodA|methodB| grep classA");
-        assertEquals(expectedTextTokenValue, actualTokens);
+        String cmd = "trace -E classA|classB methodA|methodB| grep classA";
+        List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
 
-        actualTokens = CliTokenImpl.tokenize("trace -E classA|classB methodA|methodB | grep classA");
-        assertEquals(expectedTextTokenValue, actualTokens);
+        cmd = "trace -E classA|classB methodA|methodB | grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
 
-        actualTokens = CliTokenImpl.tokenize("trace -E classA|classB methodA|methodB |grep classA");
-        assertEquals(expectedTextTokenValue, actualTokens);
+        cmd = "trace -E classA|classB methodA|methodB |grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "trace -E classA|classB methodA|methodB'|' grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "trace -E classA|classB methodA|methodB '|' grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "trace -E classA|classB methodA|methodB '|'grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+
+        cmd = "trace -E classA|classB methodA|methodB\"|\" grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "trace -E classA|classB methodA|methodB \"|\" grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "trace -E classA|classB methodA|methodB \"|\"grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB| grep", "classA"};
+        cmd = "trace -E classA|classB methodA|methodB'| 'grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "trace -E classA|classB methodA|methodB\"| \"grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB |grep", "classA"};
+        cmd = "trace -E classA|classB methodA|methodB' |'grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "trace -E classA|classB methodA|methodB\" |\"grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB '|'grep", "classA"};
+        cmd = "trace -E classA|classB methodA|methodB\" '|'\"grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB \"|\"grep", "classA"};
+        cmd = "trace -E classA|classB methodA|methodB' \"|\"'grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
     }
 
     /**
@@ -77,34 +220,247 @@ public class CliTokenImplTest {
     @Test
     public void testUnSupportedPipeChar() {
         String[] expectedTextTokenValue = new String[]{"thread|grep", "xxx"};
-        List<CliToken> actualTokens = CliTokenImpl.tokenize("thread|grep xxx");
-        assertEquals(expectedTextTokenValue, actualTokens);
+        String cmd = "thread|grep xxx";
+        List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
 
         expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB|grep", "classA"};
-        actualTokens = CliTokenImpl.tokenize("trace -E classA|classB methodA|methodB|grep classA");
-        assertEquals(expectedTextTokenValue, actualTokens);
+        cmd = "trace -E classA|classB methodA|methodB|grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
 
         expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "|", "methodA|methodB", "|", "grep", "classA"};
-        actualTokens = CliTokenImpl.tokenize("trace -E classA|classB| methodA|methodB | grep classA");
-        assertEquals(expectedTextTokenValue, actualTokens);
+        cmd = "trace -E classA|classB| methodA|methodB | grep classA";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
     }
 
-    private void assertEquals(String[] expectedTextTokenValue, List<CliToken> actualTokens) {
-        removeBlankToken(actualTokens);
-        for (int i = 0; i < expectedTextTokenValue.length; i++) {
-            Assert.assertEquals(expectedTextTokenValue[i], actualTokens.get(i).value());
-        }
+    @Test
+    public void testSeparateRedirect() {
+        String[] expectedTextTokenValue = new String[]{"jad", "aaa", ">", "bbb"};
+        String cmd = "jad aaa> bbb";
+        List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa > bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa >bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa'>' bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa '>' bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa '>'bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa\">\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+
+        cmd = "jad aaa \">\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa \">\"bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa >bbb"};
+
+        cmd = "jad aaa' >'bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa\" >\"bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa> bbb"};
+
+        cmd = "jad aaa\"> \"bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa'> 'bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa\\r", ">", "bbb"};
+
+        cmd = "jad aaa'\\r>' bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa\"\\r>\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa'", ">", "bbb"};
+        cmd = "jad aaa\"'>\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa'>'", "bbb"};
+        cmd = "jad aaa\"'>'\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa\">\"", "bbb"};
+        cmd = "jad aaa'\">\"' bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
     }
 
-    private void removeBlankToken(List<CliToken> cliTokens) {
-        CliToken blankToken = new CliTokenImpl(false, " ");
-        Iterator<CliToken> it = cliTokens.iterator();
-        while (it.hasNext()) {
-            CliToken token = it.next();
-            if (blankToken.equals(token)) {
-                it.remove();
-            }
+    @Test
+    public void testSeparateRedirectAppend() {
+        String[] expectedTextTokenValue = new String[]{"jad", "aaa", ">>", "bbb"};
+        String cmd = "jad aaa>> bbb";
+        List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa >> bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa >>bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa'>>' bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa '>>' bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa '>>'bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa\">>\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+
+        cmd = "jad aaa \">>\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa \">>\"bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa >>bbb"};
+
+        cmd = "jad aaa' >>'bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa\" >>\"bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa>> bbb"};
+
+        cmd = "jad aaa\">> \"bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa'>> 'bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa\\r", ">>", "bbb"};
+
+        cmd = "jad aaa'\\r>>' bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        cmd = "jad aaa\"\\r>>\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa'", ">>", "bbb"};
+        cmd = "jad aaa\"'>>\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa'>>'", "bbb"};
+        cmd = "jad aaa\"'>>'\" bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+
+        expectedTextTokenValue = new String[]{"jad", "aaa\">>\"", "bbb"};
+        cmd = "jad aaa'\">>\"' bbb";
+        actualTokens = CliTokenImpl.tokenize(cmd);
+        assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
+        Assert.assertEquals(cmd, concatRaw(actualTokens));
+    }
+
+    private void assertEqualsIgnoreBlank(String[] expectedTextTokenValue, List<CliToken> actualTokens) {
+        Assert.assertArrayEquals(expectedTextTokenValue, removeBlankToken(actualTokens));
+    }
+
+    private static String[] removeBlankToken(List<CliToken> cliTokens) {
+        List<CliToken> copy = new ArrayList<>(cliTokens);
+        return copy.stream()
+                .filter(token -> !token.isBlank())
+                .map(CliToken::value)
+                .toArray(String[]::new);
+    }
+
+    private static String concatRaw(List<CliToken> tokens) {
+        StringBuilder builder = new StringBuilder();
+        for (CliToken token : tokens) {
+            builder.append(token.raw());
         }
+        return builder.toString();
     }
 
 }