Fix character set handling in the MySQL lexer and query corpus

packages/mysql-parser/src/class-wp-mysql-lexer.php

@@ -667,11 +667,11 @@ private function produce_with_or_rollup( array &$out, int $with_start, int $with
 	 *
 	 * MySQL treats "@" as its own terminal. A single "@" is followed by the
 	 * unquoted user-variable name (always an identifier, never a keyword; it may
-	 * contain "." and "$"), by an empty name (MySQL's empty LEX_HOSTNAME, which
-	 * makes "SELECT @" and "user1@" valid), or by a quoted name supplied by the
-	 * following token. A "@@" prefix is two "@" terminals, followed by the system
-	 * variable name. As in MySQL, that name can't be empty, and only backticks
-	 * can quote it.
+	 * contain ".", "$", and non-ASCII characters), by an empty name (MySQL's empty
+	 * LEX_HOSTNAME, which makes "SELECT @" and "user1@" valid), or by a quoted name
+	 * supplied by the following token. A "@@" prefix is two "@" terminals, followed
+	 * by the system variable name. As in MySQL, that name can't be empty, and only
+	 * backticks can quote it.
 	 *
 	 * @param int $at_start Byte offset of the "@" token.
 	 */
@@ -697,14 +697,21 @@ private function produce_at_variable( array &$out, int $at_start ): void {
 			return;
 		}
 
-		$name_length = strspn(
-			$this->sql,
-			self::IDENTIFIER_MASK . '.',
-			$position
-		);
+		// Read the identifier characters and any "." between them as one name.
+		while ( true ) {
+			$this->read_identifier();
+			if ( '.' !== ( $this->sql[ $this->bytes_already_read ] ?? null ) ) {
+				break;
+			}
+			$this->bytes_already_read += 1;
+		}
+
+		// The "." read as a part of the name is not a qualified name separator.
+		$this->next_state = self::STATE_START;
+
+		$name_length = $this->bytes_already_read - $position;
 		if ( $name_length > 0 ) {
-			$out[]                    = $this->make_token( self::IDENT, $position, $name_length );
-			$this->bytes_already_read = $position + $name_length;
+			$out[] = $this->make_token( self::IDENT, $position, $name_length );
 		} elseif ( "'" !== $next && '"' !== $next && '`' !== $next ) {
 			$out[] = $this->make_token( self::IDENT, $position, 0 );
 		}
@@ -803,7 +810,7 @@ private function read_next_token(): ?int {
 
 		// Fast path for keywords and identifiers.
 		// `$byte > "\x7F"` catches any non-ASCII byte (0x80-0xFF); read_identifier()
-		// restricts the accepted identifier codepoints to U+0080-U+FFFF.
+		// restricts the accepted identifier codepoints to U+0080-U+10FFFF.
 		// `"'" !== $next_byte` defers x'..', n'..' and similar special
 		// literals to their dedicated branches below; only single quotes
 		// form those, regardless of SQL mode.
@@ -1026,10 +1033,13 @@ private function get_current_token_bytes(): string {
 	 *
 	 * This function reads characters that are allowed in an unquoted identifier.
 	 * It accepts digit-only sequences too; callers scan those as numbers, except
-	 * after "." or "@@".
+	 * after ".", "@", or "@@".
 	 *
 	 * Rules:
-	 *   1. Allowed characters are ASCII a-z, A-Z, 0-9, _, $, and Unicode U+0080-U+FFFF.
+	 *   1. Allowed characters are ASCII a-z, A-Z, 0-9, _, $, and Unicode U+0080-U+10FFFF.
+	 *      The MySQL documentation lists only U+0080-U+FFFF, but the MySQL lexer accepts
+	 *      any utf8mb4 character, including U+10000-U+10FFFF. MySQL then stores
+	 *      these characters as "?", because it keeps names in utf8mb3.
 	 *   2. Unquoted identifiers may begin with a digit but may not consist solely of digits.
 	 *      An exception is a part of a qualified name, such as "1" in "t.1".
 	 *
@@ -1047,13 +1057,13 @@ private function read_identifier(): ?int {
 			);
 
 			// Check if the following byte can be part of a multibyte character
-			// in the range of U+0080 to U+FFFF before looking at further bytes.
+			// in the range of U+0080 to U+10FFFF before looking at further bytes.
 			// If it can't, bail out early to avoid unnecessary UTF-8 decoding.
 			// Identifiers are usually ASCII-only, so we can optimize for that.
 			$byte_1 = ord(
 				$this->sql[ $this->bytes_already_read ] ?? "\0"
 			);
-			if ( $byte_1 < 0xC2 || $byte_1 > 0xEF ) {
+			if ( $byte_1 < 0xC2 || $byte_1 > 0xF4 ) {
 				break;
 			}
 
@@ -1086,6 +1096,24 @@ private function read_identifier(): ?int {
 				continue;
 			}
 
+			// Look for a valid 4-byte UTF-8 symbol in range U+10000 - U+10FFFF.
+			$byte_4 = ord(
+				$this->sql[ $this->bytes_already_read + 3 ] ?? "\0"
+			);
+			if (
+				$byte_1 >= 0xF0
+				&& $byte_2 >= 0x80 && $byte_2 <= 0xBF
+				&& $byte_3 >= 0x80 && $byte_3 <= 0xBF
+				&& $byte_4 >= 0x80 && $byte_4 <= 0xBF
+				// Exclude overlong encodings:
+				&& ! ( 0xF0 === $byte_1 && $byte_2 < 0x90 )
+				// Exclude codepoints above U+10FFFF:
+				&& ! ( 0xF4 === $byte_1 && $byte_2 > 0x8F )
+			) {
+				$this->bytes_already_read += 4;
+				continue;
+			}
+
 			// Not a valid identifier character.
 			break;
 		}

packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php

@@ -217,6 +217,53 @@ public function test_at_name_splits_into_at_and_ident(): void {
 		$this->assertSame( 4, $tokens[2]->length );
 	}
 
+	/**
+	 * In MySQL, the name after "@" is one token, also with non-ASCII characters.
+	 *
+	 * @dataProvider data_at_names
+	 */
+	public function test_at_name_is_one_token( string $sql, array $expected ): void {
+		$tokens = ( new WP_MySQL_Lexer( $sql ) )->remaining_tokens();
+		$this->assertSame(
+			$expected,
+			array_map(
+				static function ( WP_MySQL_Token $token ): string {
+					return $token->get_bytes();
+				},
+				$tokens
+			)
+		);
+	}
+
+	/**
+	 * @return array<string,array{0:string,1:string[]}>
+	 */
+	public function data_at_names(): array {
+		return array(
+			'ASCII'                     => array( '@a1_$', array( '@', 'a1_$', '', '' ) ),
+			'with a dot'                => array( '@a.b', array( '@', 'a.b', '', '' ) ),
+			'starting with a dot'       => array( '@.a', array( '@', '.a', '', '' ) ),
+			'ending with a dot'         => array( '@a.', array( '@', 'a.', '', '' ) ),
+			'only dots'                 => array( '@..', array( '@', '..', '', '' ) ),
+			'2-byte character'          => array( '@é', array( '@', 'é', '', '' ) ),
+			'after an ASCII character'  => array( '@xé', array( '@', 'xé', '', '' ) ),
+			'3-byte character'          => array( '@x€y', array( '@', 'x€y', '', '' ) ),
+			'4-byte character'          => array( '@🐬', array( '@', '🐬', '', '' ) ),
+			'4-byte character after x'  => array( '@x🐬', array( '@', 'x🐬', '', '' ) ),
+			'non-ASCII after a dot'     => array( '@a.🐬', array( '@', 'a.🐬', '', '' ) ),
+			'non-ASCII before a dot'    => array( '@🐬.a', array( '@', '🐬.a', '', '' ) ),
+			'followed by an operator'   => array( '@é+1', array( '@', 'é', '+', '1', '', '' ) ),
+			'followed by a name'        => array( '@é a', array( '@', 'é', 'a', '', '' ) ),
+			'followed by a number'      => array( '@a.b .5', array( '@', 'a.b', '.5', '', '' ) ),
+			'host part of a user'       => array( 'u@é.example.com', array( 'u', '@', 'é.example.com', '', '' ) ),
+
+			// Invalid UTF-8 ends the stream, as elsewhere in the lexer.
+			'invalid byte'              => array( "@x\xFF", array( '@', 'x' ) ),
+			'truncated 4-byte sequence' => array( "@x\xF0\x9F\x90", array( '@', 'x' ) ),
+			'invalid byte first'        => array( "@\xFF", array( '@', '' ) ),
+		);
+	}
+
 	public function test_at_at_splits_into_two_at_signs(): void {
 		$this->assertSame(
 			array( 'SELECT', 'AT_SIGN', 'AT_SIGN', 'IDENT', 'END_OF_INPUT', 'YYEOF' ),
@@ -425,6 +472,61 @@ public function test_identifier_utf8_three_byte_sequences(): void {
 		}
 	}
 
+	/**
+	 * Test valid and invalid 4-byte UTF-8 sequences in an identifier.
+	 * The validity is checked against PCRE with the "u" (PCRE_UTF8) modifier set.
+	 *
+	 * Testing all 4-byte sequences would take too long, so the first two bytes
+	 * are tested with valid trailing bytes, and the last two bytes are tested
+	 * with valid leading bytes. The first byte starts from 0xF0 to mark the
+	 * beginning of a 4-byte sequence, and the other bytes go from 128 to 255.
+	 */
+	public function test_identifier_utf8_four_byte_sequences(): void {
+		$values = array();
+		for ( $byte_1 = 0xF0; $byte_1 <= 0xFF; $byte_1 += 1 ) {
+			for ( $byte_2 = 128; $byte_2 <= 255; $byte_2 += 1 ) {
+				$values[] = chr( $byte_1 ) . chr( $byte_2 ) . "\x80\x80";
+			}
+		}
+		for ( $byte_3 = 128; $byte_3 <= 255; $byte_3 += 1 ) {
+			for ( $byte_4 = 128; $byte_4 <= 255; $byte_4 += 1 ) {
+				$values[] = "\xF0\x9F" . chr( $byte_3 ) . chr( $byte_4 );
+			}
+		}
+
+		foreach ( $values as $value ) {
+			$lexer  = new WP_MySQL_Lexer( $value );
+			$result = $lexer->next_token();
+			$token  = $lexer->get_token();
+
+			$is_valid = preg_match( '/^[\x{10000}-\x{10ffff}]$/u', $value );
+			if ( $is_valid ) {
+				$this->assertTrue( $result );
+				$this->assertSame( WP_MySQL_Lexer::IDENT, $token->id );
+				$this->assertSame( $value, $token->get_bytes() );
+			} else {
+				$this->assertFalse( $result );
+				$this->assertNull( $token );
+			}
+		}
+	}
+
+	public function test_identifier_with_4_byte_characters(): void {
+		// The MySQL lexer accepts 4-byte characters in unquoted identifiers on all versions.
+		foreach ( array( 50744, 80410 ) as $mysql_version ) {
+			$tokens = ( new WP_MySQL_Lexer( 'SELECT a🐬b, 𩸽 FROM 🐬', $mysql_version ) )->remaining_tokens();
+			$this->assertSame(
+				array( 'SELECT', 'a🐬b', ',', '𩸽', 'FROM', '🐬', '', '' ),
+				array_map(
+					static function ( WP_MySQL_Token $token ): string {
+						return $token->get_bytes();
+					},
+					$tokens
+				)
+			);
+		}
+	}
+
 	/**
 	 * A charset-introducer-like name used as a qualified member (after a dot)
 	 * must lex as an identifier. A real charset introducer only appears before

packages/mysql-parser/tests/WP_MySQL_Parser_Tests.php

@@ -102,9 +102,15 @@ public function test_parses_representative_statements(): void {
 			'UPDATE t SET a = a + 1 WHERE b IN (SELECT b FROM u)',
 			'DELETE FROM t WHERE a IS NOT NULL',
 			'CREATE TABLE t (id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255))',
+			'CREATE TABLE 🐬 (🐬 INT, a🐬b INT)',
+			'SELECT 🐬.a🐬b FROM 🐬 AS 𩸽',
 			'ALTER TABLE t ADD COLUMN c JSON',
 			'DROP TABLE IF EXISTS t',
 			'SET @v = 1',
+			'SET @é = 1',
+			'SELECT @xé + 1 AS plus_one',
+			'SELECT @x🐬 + 1 AS plus_one, a FROM t',
+			'CREATE USER u@é',
 			'SET @@SESSION.sql_mode = \'\'',
 			'SELECT @@`version`',
 			'SHOW TABLES',