github/arthas
4
0
Fork 0
mirror of https://github.com/alibaba/arthas.git synced 2024-04-21 10:21:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve token parsing and better support control operators such as pipe and redirections (#2658)

Browse Source
This commit is contained in:
Allan-QLB
2023-09-20 15:56:00 +08:00
committed by GitHub
parent e7baa385c0
commit eede2aacf6
2 changed files with 435 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImpl.java
+47 -26
View File
@@ -11,6 +11,9 @@ import java.util.List;
* @author <a href="mailto:julien@julienviet.com">Julien Viet</a>
*/
public class CliTokenImpl implements CliToken {
private static final String PIPE = "|";
private static final String REDIRECT = ">";
private static final String REDIRECT_APPEND = ">>";
final boolean text;
final String raw;
@@ -71,7 +74,7 @@ public class CliTokenImpl implements CliToken {
tokenize(s, 0, tokens);
tokens = correctPipeChar(tokens);
tokens = adjustTokensForSpecialSymbols(tokens);
return tokens;
}
@@ -87,35 +90,53 @@ public class CliTokenImpl implements CliToken {
* unsupported:
* 3) thread|grep xxx
* 4) trace -E classA|classB methodA|methodB|grep classA
* @param tokens
* @return
*
* Also handles redirection of '>' and '>>' in the similar way
*
* @param tokens original tokens
* @return adjusted tokens
*/
private static List<CliToken> correctPipeChar(List<CliToken> tokens) {
List<CliToken> newTokens = new ArrayList<CliToken>(tokens.size()+4);
for (CliToken token : tokens) {
String tokenValue = token.value();
if (tokenValue.length()>1 && tokenValue.endsWith("|")) {
//split last char '|'
tokenValue = tokenValue.substring(0, tokenValue.length()-1);
String rawValue = token.raw();
rawValue = rawValue.substring(0, rawValue.length()-1);
newTokens.add(new CliTokenImpl(token.isText(), rawValue, tokenValue));
//add '|' char
newTokens.add(new CliTokenImpl(true, "|", "|"));
private static List<CliToken> adjustTokensForSpecialSymbols(List<CliToken> tokens) {
return separateLeadingAndTailingSymbol(tokens, PIPE, REDIRECT_APPEND, REDIRECT);
}
} else if (tokenValue.length()>1 && tokenValue.startsWith("|")) {
//add '|' char
newTokens.add(new CliTokenImpl(true, "|", "|"));
//remove first char '|'
tokenValue = tokenValue.substring(1);
String rawValue = token.raw();
rawValue = rawValue.substring(1);
newTokens.add(new CliTokenImpl(token.isText(), rawValue, tokenValue));
} else {
newTokens.add(token);
private static List<CliToken> separateLeadingAndTailingSymbol(List<CliToken> tokens, String... symbols) {
List<CliToken> adjustedTokens = new ArrayList<>();
for (CliToken token : tokens) {
String value = token.value();
String raw = token.raw();
boolean handled = false;
for (String symbol : symbols) {
if (value.equals(symbol)) {
break;
} else if (value.endsWith(symbol)) {
handled = true;
int lastIndexOfSymbol = raw.lastIndexOf(symbol);
adjustedTokens.add(new CliTokenImpl(
token.isText(),
raw.substring(0, lastIndexOfSymbol),
value.substring(0, value.length() - symbol.length())
));
adjustedTokens.add(new CliTokenImpl(true, raw.substring(lastIndexOfSymbol), symbol));
break;
} else if (value.startsWith(symbol)) {
handled = true;
int firstIndexOfSymbol = raw.indexOf(symbol);
adjustedTokens.add(new CliTokenImpl(true,
raw.substring(0, firstIndexOfSymbol + symbol.length()), symbol));
adjustedTokens.add(new CliTokenImpl(
token.isText(),
raw.substring(firstIndexOfSymbol + symbol.length()),
value.substring(symbol.length())
));
break;
}
}
if (!handled) {
adjustedTokens.add(token);
}
}
return newTokens;
return adjustedTokens;
}
private static void tokenize(String s, int index, List<CliToken> builder) {
core/src/test/java/com/taobao/arthas/core/shell/cli/impl/CliTokenImplTest.java
+388 -32
View File
@@ -4,7 +4,7 @@ import com.taobao.arthas.core.shell.cli.CliToken;
import org.junit.Assert;
import org.junit.Test;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.List;
public class CliTokenImplTest {
@@ -25,14 +25,86 @@ public class CliTokenImplTest {
@Test
public void testSupportedPipeCharWithoutRegex() {
String[] expectedTextTokenValue = new String[]{"thread", "|", "grep", "xxx"};
List<CliToken> actualTokens = CliTokenImpl.tokenize("thread| grep xxx");
assertEquals(expectedTextTokenValue, actualTokens);
String cmd = "thread| grep xxx";
List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread | grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread |grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread'|' grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread '|' grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread '|'grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread\"|\" grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread \"|\" grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread \"|\"grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"thread| grep", "xxx"};
cmd = "thread'| 'grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread\"| \"grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"thread |grep", "xxx"};
cmd = "thread' |'grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "thread\" |\"grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"thread \"|\"grep", "xxx"};
cmd = "thread' \"|\"'grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"thread '|'grep", "xxx"};
cmd = "thread\" '|'\"grep xxx";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
actualTokens = CliTokenImpl.tokenize("thread | grep xxx");
assertEquals(expectedTextTokenValue, actualTokens);
actualTokens = CliTokenImpl.tokenize("thread |grep xxx");
assertEquals(expectedTextTokenValue, actualTokens);
}
/**
@@ -51,14 +123,85 @@ public class CliTokenImplTest {
@Test
public void testSupportedPipeCharWithRegex() {
String[] expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB", "|", "grep", "classA"};
List<CliToken> actualTokens = CliTokenImpl.tokenize("trace -E classA|classB methodA|methodB| grep classA");
assertEquals(expectedTextTokenValue, actualTokens);
String cmd = "trace -E classA|classB methodA|methodB| grep classA";
List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
actualTokens = CliTokenImpl.tokenize("trace -E classA|classB methodA|methodB | grep classA");
assertEquals(expectedTextTokenValue, actualTokens);
cmd = "trace -E classA|classB methodA|methodB | grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
actualTokens = CliTokenImpl.tokenize("trace -E classA|classB methodA|methodB |grep classA");
assertEquals(expectedTextTokenValue, actualTokens);
cmd = "trace -E classA|classB methodA|methodB |grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "trace -E classA|classB methodA|methodB'|' grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "trace -E classA|classB methodA|methodB '|' grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "trace -E classA|classB methodA|methodB '|'grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "trace -E classA|classB methodA|methodB\"|\" grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "trace -E classA|classB methodA|methodB \"|\" grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "trace -E classA|classB methodA|methodB \"|\"grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB| grep", "classA"};
cmd = "trace -E classA|classB methodA|methodB'| 'grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "trace -E classA|classB methodA|methodB\"| \"grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB |grep", "classA"};
cmd = "trace -E classA|classB methodA|methodB' |'grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "trace -E classA|classB methodA|methodB\" |\"grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB '|'grep", "classA"};
cmd = "trace -E classA|classB methodA|methodB\" '|'\"grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB \"|\"grep", "classA"};
cmd = "trace -E classA|classB methodA|methodB' \"|\"'grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
}
/**
@@ -77,34 +220,247 @@ public class CliTokenImplTest {
@Test
public void testUnSupportedPipeChar() {
String[] expectedTextTokenValue = new String[]{"thread|grep", "xxx"};
List<CliToken> actualTokens = CliTokenImpl.tokenize("thread|grep xxx");
assertEquals(expectedTextTokenValue, actualTokens);
String cmd = "thread|grep xxx";
List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "methodA|methodB|grep", "classA"};
actualTokens = CliTokenImpl.tokenize("trace -E classA|classB methodA|methodB|grep classA");
assertEquals(expectedTextTokenValue, actualTokens);
cmd = "trace -E classA|classB methodA|methodB|grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"trace", "-E", "classA|classB", "|", "methodA|methodB", "|", "grep", "classA"};
actualTokens = CliTokenImpl.tokenize("trace -E classA|classB| methodA|methodB | grep classA");
assertEquals(expectedTextTokenValue, actualTokens);
cmd = "trace -E classA|classB| methodA|methodB | grep classA";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
}
private void assertEquals(String[] expectedTextTokenValue, List<CliToken> actualTokens) {
removeBlankToken(actualTokens);
for (int i = 0; i < expectedTextTokenValue.length; i++) {
Assert.assertEquals(expectedTextTokenValue[i], actualTokens.get(i).value());
}
@Test
public void testSeparateRedirect() {
String[] expectedTextTokenValue = new String[]{"jad", "aaa", ">", "bbb"};
String cmd = "jad aaa> bbb";
List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa > bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa >bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa'>' bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa '>' bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa '>'bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa\">\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
cmd = "jad aaa \">\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa \">\"bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa >bbb"};
cmd = "jad aaa' >'bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa\" >\"bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa> bbb"};
cmd = "jad aaa\"> \"bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa'> 'bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa\\r", ">", "bbb"};
cmd = "jad aaa'\\r>' bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa\"\\r>\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa'", ">", "bbb"};
cmd = "jad aaa\"'>\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa'>'", "bbb"};
cmd = "jad aaa\"'>'\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa\">\"", "bbb"};
cmd = "jad aaa'\">\"' bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
}
private void removeBlankToken(List<CliToken> cliTokens) {
CliToken blankToken = new CliTokenImpl(false, " ");
Iterator<CliToken> it = cliTokens.iterator();
while (it.hasNext()) {
CliToken token = it.next();
if (blankToken.equals(token)) {
it.remove();
}
@Test
public void testSeparateRedirectAppend() {
String[] expectedTextTokenValue = new String[]{"jad", "aaa", ">>", "bbb"};
String cmd = "jad aaa>> bbb";
List<CliToken> actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa >> bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa >>bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa'>>' bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa '>>' bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa '>>'bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa\">>\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
cmd = "jad aaa \">>\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa \">>\"bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa >>bbb"};
cmd = "jad aaa' >>'bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa\" >>\"bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa>> bbb"};
cmd = "jad aaa\">> \"bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa'>> 'bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa\\r", ">>", "bbb"};
cmd = "jad aaa'\\r>>' bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
cmd = "jad aaa\"\\r>>\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa'", ">>", "bbb"};
cmd = "jad aaa\"'>>\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa'>>'", "bbb"};
cmd = "jad aaa\"'>>'\" bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
expectedTextTokenValue = new String[]{"jad", "aaa\">>\"", "bbb"};
cmd = "jad aaa'\">>\"' bbb";
actualTokens = CliTokenImpl.tokenize(cmd);
assertEqualsIgnoreBlank(expectedTextTokenValue, actualTokens);
Assert.assertEquals(cmd, concatRaw(actualTokens));
}
private void assertEqualsIgnoreBlank(String[] expectedTextTokenValue, List<CliToken> actualTokens) {
Assert.assertArrayEquals(expectedTextTokenValue, removeBlankToken(actualTokens));
}
private static String[] removeBlankToken(List<CliToken> cliTokens) {
List<CliToken> copy = new ArrayList<>(cliTokens);
return copy.stream()
.filter(token -> !token.isBlank())
.map(CliToken::value)
.toArray(String[]::new);
}
private static String concatRaw(List<CliToken> tokens) {
StringBuilder builder = new StringBuilder();
for (CliToken token : tokens) {
builder.append(token.raw());
}
return builder.toString();
}
}