Fix MySQL lexer correctness issues

packages/mysql-on-sqlite/src/mysql/class-wp-mysql-lexer.php

@@ -45,6 +45,7 @@ class WP_MySQL_Lexer {
 	const WHITESPACE_MASK = " \t\n\r\f\v";
 	const DIGIT_MASK      = '0123456789';
 	const HEX_DIGIT_MASK  = '0123456789abcdefABCDEF';
+	const IDENTIFIER_MASK = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$';
 
 	/**
 	 * Tokens from the MySQL Workbench "predefined.tokens" list.
@@ -943,6 +944,32 @@ class WP_MySQL_Lexer {
 	const WHITESPACE = 0;
 	const EOF        = -1;
 
+	/**
+	 * Lexer states that a token sets for the next one, like MySQL's
+	 * "next_state". They use MySQL's names from sql/sql_lex.cc, with the
+	 * abbreviations written out:
+	 *
+	 *   - STATE_START (MY_LEX_START): The default state.
+	 *   - STATE_IDENTIFIER_SEPARATOR (MY_LEX_IDENT_SEP): Set by a name before
+	 *     "." and a name character. The "." is then a separator, even before
+	 *     digits ("t.1"), and the name is an identifier, even if it's
+	 *     a keyword ("order.id").
+	 *   - STATE_IDENTIFIER_START (MY_LEX_IDENT_START): Set by a "." before
+	 *     a name character. The name is then always an identifier
+	 *     ("t.order", "t.1", "t.x'41'").
+	 *   - STATE_IDENTIFIER_OR_KEYWORD (MY_LEX_IDENT_OR_KEYWORD): Set by "@@"
+	 *     before a name character. The name is then an identifier or
+	 *     a keyword, even before "." ("@@session.x"), but never a number
+	 *     or a literal ("@@123").
+	 *
+	 * A "." that ends a number sets no state, so "1.FROM" is a number and
+	 * a keyword.
+	 */
+	private const STATE_START                 = -2;
+	private const STATE_IDENTIFIER_SEPARATOR  = -3;
+	private const STATE_IDENTIFIER_START      = -4;
+	private const STATE_IDENTIFIER_OR_KEYWORD = -5;
+
 	/**
 	 * A map of SQL keyword string values to their corresponding token types.
 	 *
@@ -2175,6 +2202,13 @@ class WP_MySQL_Lexer {
 	 */
 	private $token_type;
 
+	/**
+	 * The lexer state that the previous token set for the next one.
+	 *
+	 * @var int
+	 */
+	private $next_state = self::STATE_START;
+
 	/**
 	 * Whether the tokenizer is inside an active MySQL-specific comment.
 	 *
@@ -2419,6 +2453,23 @@ public static function get_token_name( int $token_id ): ?string {
 	}
 
 	private function read_next_token(): ?int {
+		// Read and reset the state that the previous token set.
+		$state            = $this->next_state;
+		$this->next_state = self::STATE_START;
+
+		// After "." or "@@", read a name, never a number or a literal.
+		if ( self::STATE_START !== $state ) {
+			if ( self::STATE_IDENTIFIER_START === $state ) {
+				return $this->read_identifier();
+			}
+			if ( self::STATE_IDENTIFIER_OR_KEYWORD === $state ) {
+				$type = $this->read_identifier();
+				return self::IDENTIFIER === $type
+					? $this->determine_identifier_or_keyword_type( $this->get_current_token_bytes() )
+					: $type;
+			}
+		}
+
 		$byte      = $this->sql[ $this->bytes_already_read ] ?? null;
 		$next_byte = $this->sql[ $this->bytes_already_read + 1 ] ?? null;
 
@@ -2454,18 +2505,19 @@ private function read_next_token(): ?int {
 		) {
 			$started_at = $this->bytes_already_read;
 			$type       = $this->read_identifier();
-			if (
-				self::IDENTIFIER === $type
-				// When preceded by a dot, it is always an identifier.
-				&& ! ( $started_at > 0 && '.' === $this->sql[ $started_at - 1 ] )
-			) {
+			if ( self::IDENTIFIER === $type ) {
 				// Inline the keyword lookup on the hot identifier path: most
 				// identifiers are not keywords, so this avoids two method calls
 				// (token-bytes extraction + keyword determination) per token.
 				$keyword = self::TOKENS[ strtoupper(
 					substr( $this->sql, $started_at, $this->bytes_already_read - $started_at )
 				) ] ?? self::IDENTIFIER;
-				if ( self::IDENTIFIER !== $keyword ) {
+				// Before "." and a name, a keyword is an identifier ("order.id").
+				// read_identifier() sets STATE_IDENTIFIER_SEPARATOR in that case.
+				if (
+					self::IDENTIFIER !== $keyword
+					&& self::STATE_IDENTIFIER_SEPARATOR !== $this->next_state
+				) {
 					$type = $this->resolve_keyword_type( $keyword );
 				}
 			}
@@ -2478,11 +2530,21 @@ private function read_next_token(): ?int {
 		} elseif ( null !== $byte && strspn( $byte, self::DIGIT_MASK ) > 0 ) {
 			$type = $this->read_number();
 		} elseif ( '.' === $byte ) {
-			if ( null !== $next_byte && strspn( $next_byte, self::DIGIT_MASK ) > 0 ) {
+			if (
+				null !== $next_byte
+				&& strspn( $next_byte, self::DIGIT_MASK ) > 0
+				// Right after a name, it is a qualified name separator.
+				&& self::STATE_IDENTIFIER_SEPARATOR !== $state
+			) {
 				$type = $this->read_number();
 			} else {
 				$this->bytes_already_read += 1;
 				$type                      = self::DOT_SYMBOL;
+
+				// A name character right after it starts a qualified name part.
+				if ( null !== $next_byte && ( strspn( $next_byte, self::IDENTIFIER_MASK ) > 0 || $next_byte > "\x7F" ) ) {
+					$this->next_state = self::STATE_IDENTIFIER_START;
+				}
 			}
 		} elseif ( ':' === $byte ) {
 			$this->bytes_already_read += 1; // Consume the ':'.
@@ -2603,6 +2665,17 @@ private function read_next_token(): ?int {
 			if ( '@' === $next_byte ) {
 				$this->bytes_already_read += 1; // Consume the second '@'.
 				$type                      = self::AT_AT_SIGN_SYMBOL;
+
+				// The name after it may be a keyword ("session" in "@@session.x").
+				// As in MySQL, it can't be empty, and only backticks can quote it.
+				if (
+					strspn( $this->sql, self::IDENTIFIER_MASK, $this->bytes_already_read, 1 ) > 0
+					|| ( $this->sql[ $this->bytes_already_read ] ?? '' ) > "\x7F"
+				) {
+					$this->next_state = self::STATE_IDENTIFIER_OR_KEYWORD;
+				} elseif ( '`' !== ( $this->sql[ $this->bytes_already_read ] ?? '' ) ) {
+					return null; // Invalid input.
+				}
 			} else {
 				/**
 				 * Check whether the '@' marks an unquoted user-defined variable:
@@ -2612,7 +2685,7 @@ private function read_next_token(): ?int {
 				 *   1. Starts with a '@'.
 				 *   2. Allowed following characters are ASCII a-z, A-Z, 0-9, _, ., $.
 				 */
-				$length = strspn( $this->sql, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.$', $this->bytes_already_read );
+				$length = strspn( $this->sql, self::IDENTIFIER_MASK . '.', $this->bytes_already_read );
 				if ( $length > 0 ) {
 					$this->bytes_already_read += $length;
 					$type                      = self::AT_TEXT_SUFFIX;
@@ -2644,13 +2717,9 @@ private function read_next_token(): ?int {
 		} elseif ( null === $byte ) {
 			$type = self::EOF;
 		} else {
-			$started_at = $this->bytes_already_read;
-			$type       = $this->read_identifier();
+			$type = $this->read_identifier();
 			if ( self::IDENTIFIER === $type ) {
-				// When preceded by a dot, it is always an identifier.
-				if ( $started_at > 0 && '.' === $this->sql[ $started_at - 1 ] ) {
-					$type = self::IDENTIFIER;
-				} elseif ( '_' === $byte && isset( self::UNDERSCORE_CHARSETS[ strtolower( $this->get_current_token_bytes() ) ] ) ) {
+				if ( '_' === $byte && isset( self::UNDERSCORE_CHARSETS[ strtolower( $this->get_current_token_bytes() ) ] ) ) {
 					$type = self::UNDERSCORE_CHARSET;
 				} else {
 					$type = $this->determine_identifier_or_keyword_type( $this->get_current_token_bytes() );
@@ -2672,13 +2741,13 @@ private function get_current_token_bytes(): string {
 	 * Read an unquoted identifier.
 	 *
 	 * This function reads characters that are allowed in an unquoted identifier.
-	 * An identifier cannot consist solely of digits, but this function doesn't
-	 * ensure that explicitly, as numbers are processed before identifiers in
-	 * the tokenization process, recognizing all digit-only sequences as numbers.
+	 * It accepts digit-only sequences too; callers scan those as numbers, except
+	 * after "." or "@@".
 	 *
 	 * Rules:
 	 *   1. Allowed characters are ASCII a-z, A-Z, 0-9, _, $, and Unicode U+0080-U+FFFF.
 	 *   2. Unquoted identifiers may begin with a digit but may not consist solely of digits.
+	 *      An exception is a part of a qualified name, such as "1" in "t.1".
 	 *
 	 *  See:
 	 *    https://dev.mysql.com/doc/refman/8.4/en/identifiers.html
@@ -2689,7 +2758,7 @@ private function read_identifier(): ?int {
 			// First, let's try to parse an ASCII sequence.
 			$this->bytes_already_read += strspn(
 				$this->sql,
-				'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$',
+				self::IDENTIFIER_MASK,
 				$this->bytes_already_read
 			);
 
@@ -2737,17 +2806,25 @@ private function read_identifier(): ?int {
 			break;
 		}
 
-		// An identifier cannot consist solely of digits, but we don't need to
-		// ensure that explicitly, as numbers are processed before identifiers.
+		if ( $this->bytes_already_read === $started_at ) {
+			return null; // Invalid input.
+		}
 
-		return $this->bytes_already_read - $started_at > 0
-			? self::IDENTIFIER
-			: null; // Invalid input.
+		// When "." and a name character follow, the "." is a separator.
+		// "$byte_1" already holds the byte after the name.
+		if (
+			0x2E === $byte_1
+			&& (
+				strspn( $this->sql, self::IDENTIFIER_MASK, $this->bytes_already_read + 1, 1 ) > 0
+				|| ( $this->sql[ $this->bytes_already_read + 1 ] ?? '' ) > "\x7F"
+			)
+		) {
+			$this->next_state = self::STATE_IDENTIFIER_SEPARATOR;
+		}
+		return self::IDENTIFIER;
 	}
 
 	private function read_number(): ?int {
-		// @TODO: Support numeric-only identifier parts after "." (e.g., 1ea10.1).
-
 		$byte       = $this->sql[ $this->bytes_already_read ] ?? null;
 		$next_byte  = $this->sql[ $this->bytes_already_read + 1 ] ?? null;
 		$third_byte = $this->sql[ $this->bytes_already_read + 2 ] ?? null;

packages/mysql-on-sqlite/src/sqlite/class-wp-sqlite-information-schema-builder.php

@@ -3324,8 +3324,9 @@ private function get_value( WP_Parser_Node $node ): string {
 	 * This is used for storing DEFAULT and CHECK expressions in the database.
 	 *
 	 * The current implementation is using a naive approach based on directly
-	 * joining the original expression token bytes. This is safe, beacuase the
-	 * original tokens must comprise a valid expression. While functionally
+	 * joining the original expression token bytes. This is safe, because the
+	 * original tokens must comprise a valid expression, and they are joined so
+	 * that they are lexed the same way when parsed again. While functionally
 	 * equivalent, it is not strictly identical to what MySQL stores, because
 	 * MySQL normalizes and prints the expression in a specific format.
 	 *
@@ -3347,23 +3348,22 @@ private function serialize_mysql_expression( WP_Parser_Node $node ): string {
 		}
 
 		$value         = '';
-		$last_token_id = null;
-		foreach ( $node->get_descendant_tokens() as $i => $token ) {
-			// Do not insert whitespace around parentheses. This is primarily to
-			// avoid inserting whitespace before '(', which may break function
-			// calls, depending on the value of the "IGNORE_SPACE" SQL mode.
-			if (
-				0 === $i
-				|| WP_MySQL_Lexer::OPEN_PAR_SYMBOL === $token->id
+		$last_no_space = true;
+		foreach ( $node->get_descendant_tokens() as $token ) {
+			// Do not insert whitespace around parentheses and dots, and after "@@".
+			// Before '(', whitespace may break function calls, depending on the
+			// value of the "IGNORE_SPACE" SQL mode. Around '.', it changes how
+			// qualified names are lexed: "t . 1" has a number, and "order . id"
+			// has a keyword. After "@@", it leaves the variable name empty.
+			$no_space = WP_MySQL_Lexer::OPEN_PAR_SYMBOL === $token->id
 				|| WP_MySQL_Lexer::CLOSE_PAR_SYMBOL === $token->id
-				|| WP_MySQL_Lexer::OPEN_PAR_SYMBOL === $last_token_id
-				|| WP_MySQL_Lexer::CLOSE_PAR_SYMBOL === $last_token_id
-			) {
-				$value .= $token->get_bytes();
-			} else {
-				$value .= ' ' . $token->get_bytes();
+				|| WP_MySQL_Lexer::DOT_SYMBOL === $token->id;
+
+			if ( ! $no_space && ! $last_no_space ) {
+				$value .= ' ';
 			}
-			$last_token_id = $token->id;
+			$value        .= $token->get_bytes();
+			$last_no_space = $no_space || WP_MySQL_Lexer::AT_AT_SIGN_SYMBOL === $token->id;
 		}
 		return $value;
 	}

packages/mysql-on-sqlite/tests/WP_MySQL_On_SQLite_Tests.php

@@ -14475,6 +14475,101 @@ public function testFullyQualifiedTableName(): void {
 		$this->assertCount( 0, $result );
 	}
 
+	public function testQualifiedNamePartsStartingWithDigits(): void {
+		$this->assertQuery( 'CREATE TABLE 1t (`1` INT, `1st` INT, `1e5` INT)' );
+		$this->assertQuery( 'INSERT INTO 1t VALUES (1, 2, 3), (4, 5, 6)' );
+
+		$result = $this->assertQuery( 'SELECT 1t.1, 1t.1st, 1t.1e5 FROM 1t WHERE 1t.1 = 4' );
+		$this->assertEquals(
+			array(
+				(object) array(
+					'1'   => '4',
+					'1st' => '5',
+					'1e5' => '6',
+				),
+			),
+			$result
+		);
+
+		$result = $this->assertQuery( 'SELECT wp.1t.1st FROM wp.1t ORDER BY wp.1t.1 DESC' );
+		$this->assertEquals(
+			array( (object) array( '1st' => '5' ), (object) array( '1st' => '2' ) ),
+			$result
+		);
+	}
+
+	public function testReservedWordsAsQualifiers(): void {
+		$this->assertQuery( 'CREATE TABLE `order` (id INT, `1` INT)' );
+		$this->assertQuery( 'INSERT INTO `order` VALUES (7, 8), (9, 10)' );
+
+		$result = $this->assertQuery( 'SELECT order.id, order.1 FROM `order` WHERE order.id = 7' );
+		$this->assertEquals(
+			array(
+				(object) array(
+					'id' => '7',
+					'1'  => '8',
+				),
+			),
+			$result
+		);
+
+		$result = $this->assertQuery( 'SELECT wp.order.1 FROM wp.`order` ORDER BY order.id DESC' );
+		$this->assertEquals(
+			array( (object) array( '1' => '10' ), (object) array( '1' => '8' ) ),
+			$result
+		);
+	}
+
+	public function testCheckConstraintsWithQualifiedNames(): void {
+		// The stored check clauses are parsed again to create the SQLite table.
+		$this->assertQuery( 'CREATE TABLE `order` (id INT, `1` INT, CHECK (order.id > 0), CHECK (wp.order.1 < 10))' );
+
+		$result = $this->assertQuery( 'SHOW CREATE TABLE `order`' );
+		$this->assertEquals(
+			implode(
+				"\n",
+				array(
+					'CREATE TABLE `order` (',
+					'  `id` int DEFAULT NULL,',
+					'  `1` int DEFAULT NULL,',
+					'  CONSTRAINT `order_chk_1` CHECK (order.id > 0),',
+					'  CONSTRAINT `order_chk_2` CHECK (wp.order.1 < 10)',
+					') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci',
+				)
+			),
+			$result[0]->{'Create Table'}
+		);
+
+		$this->assertQuery( 'INSERT INTO `order` VALUES (1, 9)' );
+
+		$this->expectException( WP_MySQL_On_SQLite_Exception::class );
+		$this->expectExceptionMessage( 'SQLSTATE[23000]: Integrity constraint violation: 19 CHECK constraint failed: order_chk_2' );
+		$this->assertQuery( 'INSERT INTO `order` VALUES (1, 10)' );
+	}
+
+	public function testCheckConstraintsAndDefaultsWithSystemVariables(): void {
+		// MySQL rejects variables in CHECK and DEFAULT expressions, but the driver
+		// accepts them. ALTER TABLE parses the stored expressions again.
+		$this->assertQuery( 'CREATE TABLE t (a INT DEFAULT (@@auto_increment_increment), b INT, CHECK (b < @@max_connections))' );
+		$this->assertQuery( 'ALTER TABLE t ADD COLUMN c INT' );
+
+		$result = $this->assertQuery( 'SHOW CREATE TABLE t' );
+		$this->assertEquals(
+			implode(
+				"\n",
+				array(
+					'CREATE TABLE `t` (',
+					'  `a` int DEFAULT (@@auto_increment_increment),',
+					'  `b` int DEFAULT NULL,',
+					'  `c` int DEFAULT NULL,',
+					'  CONSTRAINT `t_chk_1` CHECK (b < @@max_connections)',
+					') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci',
+				)
+			),
+			$result[0]->{'Create Table'}
+		);
+	}
+
 	public function testWriteWithUsageOfInformationSchemaTables(): void {
 		// Ensure "information_schema.tables" is empty.
 		$this->assertQuery( 'DROP TABLE _options, _dates' );