Skip to content

Commit 6c71e15

Browse files
tanjiroK-codertanjiroK-coder
authored
avoid quadratic getToken(i) rescans in lookahead token walks (#2862)
1 parent 5d07ba4 commit 6c71e15

2 files changed

Lines changed: 60 additions & 53 deletions

File tree

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 43 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -425,11 +425,10 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
425425
Token token = getToken(offset);
426426
switch (token.kind) {
427427
case K_SET:
428-
int nameEnd = offset + 2;
429-
while (".".equals(getToken(nameEnd).image)) {
430-
nameEnd += 2;
428+
Token assignment = getToken(offset + 2);
429+
while (".".equals(assignment.image)) {
430+
assignment = nextWithItemToken(nextWithItemToken(assignment));
431431
}
432-
Token assignment = getToken(nameEnd);
433432
return assignment.kind == K_TO || "=".equals(assignment.image)
434433
|| assignment.kind == K_FROM;
435434
case K_RESET:
@@ -521,15 +520,14 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
521520
boolean statementStart = true;
522521
boolean statements = false;
523522
int caseDepth = 0;
524-
for (int i = 2; ; i++) {
525-
Token token = getToken(i);
523+
for (Token token = getToken(2); ; token = nextWithItemToken(token)) {
526524
if (token.kind == EOF) {
527525
return false;
528526
} else if (token.kind == ST_SEMICOLON) {
529527
statementStart = true;
530528
} else if (statementStart) {
531529
if (token.kind == K_END) {
532-
int next = getToken(i + 1).kind;
530+
int next = nextWithItemToken(token).kind;
533531
return statements && (next == ST_SEMICOLON || next == EOF);
534532
}
535533
switch (token.kind) {
@@ -676,8 +674,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
676674
try {
677675
int depth = 0;
678676
Token prev = null;
679-
for (int i = 2; ; i++) {
680-
Token t = getToken(i);
677+
for (Token t = getToken(2); ; t = nextWithItemToken(t)) {
681678
if (t == null || t.kind == EOF) {
682679
return false;
683680
}
@@ -841,8 +838,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
841838
// CHAIN_CACHE_THRESHOLD delimiter/part pairs (only in the non-filling pass).
842839
private int isFunctionAheadWalk(boolean fill) {
843840
try {
844-
int i = 1;
845-
Token t = getToken(i);
841+
Token t = getToken(1);
846842

847843
// JDBC escape function: {fn ...} — must check for FN keyword
848844
if (t.image.equals("{")) {
@@ -851,8 +847,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
851847

852848
// Optional APPROXIMATE keyword
853849
if (t.kind == K_APPROXIMATE) {
854-
i++;
855-
t = getToken(i);
850+
t = getToken(2);
856851
}
857852

858853
// Exclude tokens that have their own dedicated branches
@@ -867,32 +862,31 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
867862
|| t.kind == CLOSING_BRACKET || t.kind == EOF || "[".equals(t.image)) {
868863
return 0;
869864
}
870-
i++;
871-
872-
// Walk through dotted name chain
865+
// Walk through dotted name chain, following the token chain directly;
866+
// repeated getToken(i) calls rescan from the start each time
873867
int delimiters = 0;
874868
List<Token> chainParts = fill ? new ArrayList<Token>() : null;
869+
t = nextWithItemToken(t);
875870
while (true) {
876-
t = getToken(i);
877871
if (t.image.equals(".") || t.image.equals("..")
878872
|| t.image.equals("...") || t.image.equals(":")) {
879873
if (!fill && ++delimiters > CHAIN_CACHE_THRESHOLD) {
880874
return LONG_CHAIN;
881875
}
876+
Token part = nextWithItemToken(t); // skip delimiter
882877
if (fill) {
883-
chainParts.add(getToken(i + 1));
878+
chainParts.add(part);
884879
}
885-
i++; // skip delimiter
886-
i++; // skip next name part
880+
t = nextWithItemToken(part); // skip next name part
887881
} else {
888882
break;
889883
}
890884
}
891885

892886
// Must be followed by (, and not by the Oracle join syntax column(+)
893-
boolean result = getToken(i).kind == OPENING_BRACKET
894-
&& !(getToken(i + 1).image.equals("+")
895-
&& getToken(i + 2).kind == CLOSING_BRACKET);
887+
boolean result = t.kind == OPENING_BRACKET
888+
&& !(nextWithItemToken(t).image.equals("+")
889+
&& nextWithItemToken(nextWithItemToken(t)).kind == CLOSING_BRACKET);
896890

897891
// Each eligible chain part would walk the same remaining suffix and
898892
// obtain the same answer, so memoize them all in one pass.
@@ -1492,41 +1486,38 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
14921486
// Returns 1 for true, 0 for false, or LONG_CHAIN when the walk exceeded
14931487
// CHAIN_CACHE_THRESHOLD delimiter/part pairs (only in the non-filling pass).
14941488
private int isAllTableColumnsWalk(boolean fill) {
1495-
int i = 1;
1496-
Token t = getToken(i);
1489+
Token last = getToken(1);
14971490

14981491
// Must start with a name-like token
1499-
if (t.kind == S_LONG || t.kind == S_DOUBLE || t.kind == S_HEX
1500-
|| t.kind == S_CHAR_LITERAL || t.kind == OPENING_BRACKET
1501-
|| t.kind == CLOSING_BRACKET || t.kind == EOF) {
1492+
if (last.kind == S_LONG || last.kind == S_DOUBLE || last.kind == S_HEX
1493+
|| last.kind == S_CHAR_LITERAL || last.kind == OPENING_BRACKET
1494+
|| last.kind == CLOSING_BRACKET || last.kind == EOF) {
15021495
return 0;
15031496
}
1504-
i++;
15051497

1506-
// Walk through dotted name chain
1498+
// Walk through dotted name chain, following the token chain directly;
1499+
// repeated getToken(i) calls rescan from the start each time
15071500
int delimiters = 0;
15081501
List<Token> chainParts = fill ? new ArrayList<Token>() : null;
15091502
while (true) {
1510-
t = getToken(i);
1503+
Token t = nextWithItemToken(last);
15111504
if (t.image.equals(".") || t.image.equals("..")
15121505
|| t.image.equals("...")) {
15131506
if (!fill && ++delimiters > CHAIN_CACHE_THRESHOLD) {
15141507
return LONG_CHAIN;
15151508
}
1509+
last = nextWithItemToken(t); // skip delimiter, take next part (could be "*")
15161510
if (fill) {
1517-
chainParts.add(getToken(i + 1));
1511+
chainParts.add(last);
15181512
}
1519-
i++; // skip delimiter
1520-
i++; // skip next part (could be "*")
15211513
} else {
15221514
break;
15231515
}
15241516
}
15251517

15261518
// It's AllTableColumns if the chain ended on "*"
15271519
// i.e., the last name part we skipped over was "*"
1528-
// Back up: the last token consumed was at (i-1)
1529-
boolean result = getToken(i - 1).image.equals("*");
1520+
boolean result = last.image.equals("*");
15301521

15311522
// Each chain part that would pass the name-like guard walks the same
15321523
// remaining suffix and obtains the same answer.
@@ -2721,9 +2712,8 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
27212712
return false;
27222713
}
27232714
int depth = 1;
2724-
int i = 4;
27252715
while (depth > 0) {
2726-
t = getToken(i++);
2716+
t = nextWithItemToken(t);
27272717
if (t == null || t.kind == EOF) {
27282718
return false;
27292719
}
@@ -2733,7 +2723,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
27332723
depth--;
27342724
}
27352725
}
2736-
t = getToken(i);
2726+
t = nextWithItemToken(t);
27372727
if (t.kind == K_DO) {
27382728
return true;
27392729
}
@@ -2742,8 +2732,8 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
27422732
}
27432733
// the index predicate ends where the conflict action's DO begins
27442734
depth = 0;
2745-
for (i++; ; i++) {
2746-
t = getToken(i);
2735+
while (true) {
2736+
t = nextWithItemToken(t);
27472737
if (t == null || t.kind == EOF) {
27482738
return false;
27492739
}
@@ -2874,11 +2864,14 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
28742864
* user@host, which the lexer reads as one identifier, counts only when merged is set.
28752865
*/
28762866
private boolean isMySqlAccountHostAhead(int offset, boolean merged) {
2877-
Token name = getToken(offset);
2867+
return isMySqlAccountHost(getToken(offset), merged);
2868+
}
2869+
2870+
private boolean isMySqlAccountHost(Token name, boolean merged) {
28782871
if (name.kind == S_IDENTIFIER && name.image.indexOf('@') > 0) {
28792872
return merged || name.image.endsWith("@");
28802873
}
2881-
Token next = getToken(offset + 1);
2874+
Token next = nextWithItemToken(name);
28822875
return next.kind == K_AT_SIGN
28832876
|| next.kind == S_AT_IDENTIFIER && !next.image.startsWith("@@");
28842877
}
@@ -2901,9 +2894,9 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
29012894
if (isMySqlDialect()) { return true; }
29022895
if (next.kind == K_ALL) { return getToken(4).kind == K_EXCEPT; }
29032896
// role [@ host] [, role [@ host]] ...
2904-
for (int i = 3; ; i += 2) {
2905-
if (isMySqlAccountHostAhead(i, false)) { return true; }
2906-
if (!",".equals(getToken(i + 1).image)) { return false; }
2897+
for (Token role = next; ; role = nextWithItemToken(nextWithItemToken(role))) {
2898+
if (isMySqlAccountHost(role, false)) { return true; }
2899+
if (!",".equals(nextWithItemToken(role).image)) { return false; }
29072900
}
29082901
}
29092902

@@ -2937,8 +2930,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
29372930

29382931
private boolean isObjectPrivilegeAhead() {
29392932
int depth = 0;
2940-
for (int i = 1; ; i++) {
2941-
Token token = getToken(i);
2933+
for (Token token = getToken(1); ; token = nextWithItemToken(token)) {
29422934
if (token.kind == EOF || ";".equals(token.image)) { return false; }
29432935
if ("(".equals(token.image)) { depth++; }
29442936
else if (")".equals(token.image)) { depth--; }
@@ -2983,10 +2975,9 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
29832975
private boolean isPostgreSqlTriggerAhead() {
29842976
if (getToken(1).kind == K_CONSTRAINT) { return true; }
29852977
if (getToken(1).kind != K_TRIGGER) { return false; }
2986-
for (int i = 2; ; i++) {
2987-
Token token = getToken(i);
2978+
for (Token token = getToken(2); ; token = nextWithItemToken(token)) {
29882979
if (token.kind == EOF || ";".equals(token.image) || token.kind == K_BEGIN) { return false; }
2989-
if (token.kind == K_EXECUTE && (getToken(i + 1).kind == K_FUNCTION || getToken(i + 1).kind == K_PROCEDURE)) { return true; }
2980+
if (token.kind == K_EXECUTE && (nextWithItemToken(token).kind == K_FUNCTION || nextWithItemToken(token).kind == K_PROCEDURE)) { return true; }
29902981
// DuckDB 2.0 uses the standard header with a statement body instead of EXECUTE
29912982
if ("REFERENCING".equalsIgnoreCase(token.image) || token.kind == K_STATEMENT) { return true; }
29922983
}
@@ -3404,8 +3395,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
34043395
*/
34053396
protected boolean isNamedExprListAhead() {
34063397
int depth = 0;
3407-
for (int i = 1; ; i++) {
3408-
Token t = getToken(i);
3398+
for (Token t = getToken(1); ; t = nextWithItemToken(t)) {
34093399
if (t.kind == EOF) {
34103400
return false;
34113401
}

‎src/test/java/net/sf/jsqlparser/parser/LongChainPredicateTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,11 @@
1111

1212
import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed;
1313
import static org.junit.jupiter.api.Assertions.assertFalse;
14+
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
1415
import static org.junit.jupiter.api.Assertions.assertTrue;
1516

17+
import java.time.Duration;
18+
1619
import net.sf.jsqlparser.expression.Function;
1720
import net.sf.jsqlparser.schema.Column;
1821
import net.sf.jsqlparser.statement.select.AllTableColumns;
@@ -100,4 +103,18 @@ void oracleOuterJoinColumnPlusOnMemoPath() throws Exception {
100103
assertSqlCanBeParsedAndDeparsed(
101104
"SELECT * FROM a, b WHERE " + chain(8, "x(+)") + " = b.x");
102105
}
106+
107+
@Test
108+
void veryLongChainsParseInLinearTime() {
109+
// the memoizing walk used to fetch each part with getToken(i), which rescans the
110+
// token list from the start, so one walk over a 10000-part chain took many seconds
111+
assertTimeoutPreemptively(Duration.ofSeconds(5), () -> {
112+
assertTrue(
113+
firstExpression("SELECT " + chain(10000, "col") + " FROM t") instanceof Column);
114+
assertTrue(firstExpression(
115+
"SELECT " + chain(10000, "*") + " FROM t") instanceof AllTableColumns);
116+
assertTrue(firstExpression(
117+
"SELECT " + chain(10000, "f(1)") + " FROM t") instanceof Function);
118+
});
119+
}
103120
}

0 commit comments

Comments
 (0)