Skip to content

Commit

Permalink
Use WP coding styles for filenames, use file per class, improve direc…
Browse files Browse the repository at this point in the history
…tory structure
  • Loading branch information
JanJakes committed Oct 31, 2024
1 parent 7071e6f commit 17b2613
Show file tree
Hide file tree
Showing 14 changed files with 564 additions and 519 deletions.
448 changes: 0 additions & 448 deletions custom-parser/parser/DynamicRecursiveDescentParser.php

This file was deleted.

File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
<?php

require_once __DIR__ . '/../parser/MySQLLexer.php';
require_once __DIR__ . '/../../wp-includes/mysql/class-mysql-lexer.php';

const GRAMMAR_FILE = __DIR__ . '/../parser/grammar.php';
const GRAMMAR_FILE = __DIR__ . '/../../wp-includes/mysql/mysql-grammar.php';

// Convert the original MySQLParser.g4 grammar to a JSON format.
// The grammar is also flattened and expanded to an ebnf-to-json-like format.
Expand Down Expand Up @@ -240,7 +240,7 @@ function export_as_php_var( $variable ) {
foreach ( $branch as $i => $name ) {
$is_terminal = ! isset( $rule_id_by_name[ $name ] );
if ( $is_terminal ) {
$new_branch[] = MySQLLexer::get_token_id( $name );
$new_branch[] = MySQL_Lexer::get_token_id( $name );
} else {
// Use rule id to avoid conflicts with token ids
$new_branch[] = $rule_id_by_name[ $name ];
Expand Down
3 changes: 1 addition & 2 deletions tests/parser/run-lexer-tests.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ function ( $severity, $message, $file, $line ) {
}
);

require_once __DIR__ . '/../../custom-parser/parser/DynamicRecursiveDescentParser.php';
require_once __DIR__ . '/../../custom-parser/parser/MySQLLexer.php';
require_once __DIR__ . '/../../wp-includes/mysql/class-mysql-lexer.php';

$handle = fopen( __DIR__ . '/data/queries.csv', 'r' );

Expand Down
10 changes: 5 additions & 5 deletions tests/parser/run-parser-tests.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ function ( $severity, $message, $file, $line ) {
}
);

require_once __DIR__ . '/../../wp-includes/mysql/class-mysql-lexer.php';
require_once __DIR__ . '/../../wp-includes/parser/class-parser.php';

function getStats( $total, $failures, $exceptions ) {
return sprintf(
'Total: %5d | Failures: %4d / %2d%% | Exceptions: %4d / %2d%%',
Expand All @@ -18,10 +21,7 @@ function getStats( $total, $failures, $exceptions ) {
);
}

require_once __DIR__ . '/../../custom-parser/parser/DynamicRecursiveDescentParser.php';
require_once __DIR__ . '/../../custom-parser/parser/MySQLLexer.php';

$grammar_data = include __DIR__ . '/../../custom-parser/parser/grammar.php';
$grammar_data = include __DIR__ . '/../../wp-includes/mysql/mysql-grammar.php';
$grammar = new Grammar( $grammar_data );

$handle = fopen( __DIR__ . '/data/queries.csv', 'r' );
Expand Down Expand Up @@ -50,7 +50,7 @@ function getStats( $total, $failures, $exceptions ) {
throw new Exception( 'Empty tokens' );
}

$parser = new DynamicRecursiveDescentParser( $grammar, $tokens );
$parser = new Parser( $grammar, $tokens );
$parse_tree = $parser->parse();
if ( null === $parse_tree ) {
$failures[] = $query;
Expand Down
48 changes: 48 additions & 0 deletions tests/parser/run-single.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
<?php

// throw exception if anything fails
set_error_handler(
function ( $severity, $message, $file, $line ) {
throw new ErrorException( $message, 0, $severity, $file, $line );
}
);

require_once __DIR__ . '/../../wp-includes/mysql/class-mysql-lexer.php';
require_once __DIR__ . '/../../wp-includes/parser/class-parser.php';

$grammar_data = include __DIR__ . '/../../wp-includes/mysql/mysql-grammar.php';
$grammar = new Grammar( $grammar_data );


//$tokens = tokenizeQuery('SELECT x FROM t');
//$tokens = tokenizeQuery('ALTER TABLE t1 ORDER BY t1.id');
//$tokens = tokenizeQuery("SELECT CAST(1988 AS YEAR)");
//$tokens = tokenizeQuery("DROP FUNCTION x");
//$tokens = tokenizeQuery("SELECT ROW_NUMBER() OVER (PARTITION BY g)");
//$tokens = tokenizeQuery("GRANT ALL ON db.t1 TO 'testuser'@'localhost'");
//$tokens = tokenizeQuery("SELECT id max FROM t");
//$tokens = tokenizeQuery("CREATE TABLE t (tmst TIMESTAMP DEFAULT NOW())");
//$tokens = tokenizeQuery("SELECT CURRENT_TIMESTAMP");
//$tokens = tokenizeQuery("SELECT TIMESTAMP'2015-01-01 12:00:00'");
//$tokens = tokenizeQuery("create table `#mysql50#abc``def` ( id int )");
//$tokens = tokenizeQuery("INSERT INTO `±íÒ»` VALUES (' !\"\"#$%&\'()*+,-./');");
//$tokens = tokenizeQuery("SELECT ADDDATE(DATE'2021-01-01', INTERVAL 1 DAY)");
//$tokens = tokenizeQuery("select 1ea10.1a20,1e+ 1e+10 from 1ea10");
//$tokens = tokenizeQuery("ALTER TABLE t2 ALTER my_row_id SET VISIBLE");
//$tokens = tokenizeQuery("ANALYZE TABLE t UPDATE HISTOGRAM ON col1 USING DATA ''");
//$tokens = tokenizeQuery("((SELECT 1 UNION SELECT 1) UNION SELECT 2)");
//$tokens = tokenizeQuery("alter user mysqltest_7@");
//$tokens = tokenizeQuery("SELECT * FROM t0 WHERE a < ANY (VALUES ROW(1))");
//$tokens = tokenizeQuery("SELECT 1 /*!99999 /* */ */");
//$tokens = tokenizeQuery("SHOW GRANTS FOR u1 using r1");
//$tokens = tokenizeQuery("(SELECT 1) LIMIT 1 INTO @var");
//$tokens = tokenizeQuery("DO ST_AsText(@centroid_point)");
//$tokens = tokenizeQuery("ALTER SCHEMA s1 READ ONLY DEFAULT");
$tokens = tokenize_query( 'CREATE TABLE t1 (g MULTIPOINT)' );

foreach ( $tokens as $token ) {
echo $token, PHP_EOL;
}
$parser = new Parser( $grammar, $tokens );
$parse_tree = $parser->parse();
var_dump( $parse_tree );
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<?php
<?php // phpcs:disable WordPress.Files.FileName.InvalidClassFileName,Generic.Files.OneObjectStructurePerFile.MultipleFound

class SQLiteTokenFactory {
private static $valid_types = array(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<?php
<?php // phpcs:disable WordPress.Files.FileName.InvalidClassFileName

require_once __DIR__ . '/MySQLLexer.php';
require_once __DIR__ . '/DynamicRecursiveDescentParser.php';
Expand Down Expand Up @@ -62,7 +62,7 @@ public function run_query( $query ) {
$this->has_found_rows_call = false;
$this->last_calc_rows_result = null;

$parser = new DynamicRecursiveDescentParser( $this->grammar, tokenize_query( $query ) );
$parser = new Parser( $this->grammar, tokenize_query( $query ) );
$parse_tree = $parser->parse();
$expr = $this->translate_query( $parse_tree );
$expr = $this->rewrite_sql_calc_found_rows( $expr );
Expand Down Expand Up @@ -126,13 +126,13 @@ private function translate_query( $parse_tree ) {
return null;
}

if ( $parse_tree instanceof MySQLToken ) {
if ( $parse_tree instanceof MySQL_Token ) {
$token = $parse_tree;
switch ( $token->type ) {
case MySQLLexer::EOF:
case MySQL_Lexer::EOF:
return new SQLiteExpression( array() );

case MySQLLexer::IDENTIFIER:
case MySQL_Lexer::IDENTIFIER:
return new SQLiteExpression(
array(
SQLiteTokenFactory::identifier(
Expand All @@ -150,7 +150,7 @@ private function translate_query( $parse_tree ) {
}
}

if ( ! ( $parse_tree instanceof ParseTree ) ) {
if ( ! ( $parse_tree instanceof Parse_Tree ) ) {
throw new Exception( 'translateQuery only accepts MySQLToken and ParseTree instances' );
}

Expand All @@ -165,14 +165,14 @@ private function translate_query( $parse_tree ) {
case 'querySpecOption':
$token = $parse_tree->get_token();
switch ( $token->type ) {
case MySQLLexer::ALL_SYMBOL:
case MySQLLexer::DISTINCT_SYMBOL:
case MySQL_Lexer::ALL_SYMBOL:
case MySQL_Lexer::DISTINCT_SYMBOL:
return new SQLiteExpression(
array(
SQLiteTokenFactory::raw( $token->text ),
)
);
case MySQLLexer::SQL_CALC_FOUND_ROWS_SYMBOL:
case MySQL_Lexer::SQL_CALC_FOUND_ROWS_SYMBOL:
$this->has_sql_calc_found_rows = true;
// Fall through to default.
default:
Expand All @@ -188,7 +188,7 @@ private function translate_query( $parse_tree ) {
// FROM DUAL statement, as FROM mytable, DUAL is a syntax
// error.
if (
$parse_tree->has_token( MySQLLexer::DUAL_SYMBOL ) &&
$parse_tree->has_token( MySQL_Lexer::DUAL_SYMBOL ) &&
! $parse_tree->has_child( 'tableReferenceList' )
) {
return null;
Expand Down Expand Up @@ -272,10 +272,10 @@ private function translate_query( $parse_tree ) {
case 'textStringLiteral':
return new SQLiteExpression(
array(
$parse_tree->has_token( MySQLLexer::DOUBLE_QUOTED_TEXT ) ?
SQLiteTokenFactory::double_quoted_value( $parse_tree->get_token( MySQLLexer::DOUBLE_QUOTED_TEXT )->text ) : false,
$parse_tree->has_token( MySQLLexer::SINGLE_QUOTED_TEXT ) ?
SQLiteTokenFactory::raw( $parse_tree->get_token( MySQLLexer::SINGLE_QUOTED_TEXT )->text ) : false,
$parse_tree->has_token( MySQL_Lexer::DOUBLE_QUOTED_TEXT ) ?
SQLiteTokenFactory::double_quoted_value( $parse_tree->get_token( MySQL_Lexer::DOUBLE_QUOTED_TEXT )->text ) : false,
$parse_tree->has_token( MySQL_Lexer::SINGLE_QUOTED_TEXT ) ?
SQLiteTokenFactory::raw( $parse_tree->get_token( MySQL_Lexer::SINGLE_QUOTED_TEXT )->text ) : false,
)
);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
<?php

require_once __DIR__ . '/class-mysql-token.php';

/**
* This lexer is based on the MySQL Workbench lexer grammar.
* See:
* https://github.com/mysql/mysql-workbench/blob/8.0.38/library/parsers/grammars/MySQLLexer.g4
* https://github.com/mysql/mysql-workbench/blob/8.0.38/library/parsers/grammars/predefined.tokens
* https://github.com/mysql/mysql-workbench/blob/8.0.38/library/parsers/mysql/MySQLBaseLexer.cpp
*/
class MySQLLexer {
class MySQL_Lexer {
// Token channels
const CHANNEL_DEFAULT = 0;
const CHANNEL_HIDDEN = 99;
Expand Down Expand Up @@ -2391,14 +2393,14 @@ private function next_token() {
}
} elseif ( null === $la ) {
$this->match_eof();
$this->token_instance = new MySQLToken( self::EOF, '<EOF>' );
$this->token_instance = new MySQL_Token( self::EOF, '<EOF>' );
return false;
} else {
$this->consume();
$this->type = self::INVALID_INPUT;
}

$this->token_instance = null === $this->type ? null : new MySQLToken( $this->type, $this->text, $this->channel );
$this->token_instance = null === $this->type ? null : new MySQL_Token( $this->type, $this->text, $this->channel );
return true;
}

Expand Down Expand Up @@ -2476,7 +2478,7 @@ protected function identifier_or_keyword() {
}

// With "SQL_MODE_HIGH_NOT_PRECEDENCE" enabled, "NOT" needs to be emitted as a higher priority NOT2 symbol.
if ( self::NOT_SYMBOL === $this->type && $this->is_sql_mode_active( MySQLLexer::SQL_MODE_HIGH_NOT_PRECEDENCE ) ) {
if ( self::NOT_SYMBOL === $this->type && $this->is_sql_mode_active( MySQL_Lexer::SQL_MODE_HIGH_NOT_PRECEDENCE ) ) {
$this->type = self::NOT2_SYMBOL;
}

Expand Down Expand Up @@ -2739,46 +2741,12 @@ private function determine_numeric_type( $text ) {
}
}

class MySQLToken {
public $type;
public $text;
public $channel;

public function __construct( $type, $text, $channel = null ) {
$this->type = $type;
$this->text = $text;
$this->channel = $channel;
}

public function get_type() {
return $this->type;
}

public function get_name() {
return MySQLLexer::get_token_name( $this->type );
}

public function get_text() {
return $this->text;
}

public function get_channel() {
return $this->channel;
}

public function __toString() {
return $this->text . '<' . $this->type . ',' . $this->get_name() . '>';
}

public function extract_value() {
if ( MySQLLexer::BACK_TICK_QUOTED_ID === $this->type ) {
return substr( $this->text, 1, -1 );
} elseif ( MySQLLexer::DOUBLE_QUOTED_TEXT === $this->type ) {
return substr( $this->text, 1, -1 );
} elseif ( MySQLLexer::SINGLE_QUOTED_TEXT === $this->type ) {
return substr( $this->text, 1, -1 );
} else {
return $this->text;
}
}
function tokenize_query( $sql ) {
$lexer = new MySQL_Lexer( $sql );
$tokens = array();
do {
$token = $lexer->get_next_token();
$tokens[] = $token;
} while ( MySQL_Lexer::EOF !== $token->type );
return $tokens;
}
45 changes: 45 additions & 0 deletions wp-includes/mysql/class-mysql-token.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
<?php

class MySQL_Token {
public $type;
public $text;
public $channel;

public function __construct( $type, $text, $channel = null ) {
$this->type = $type;
$this->text = $text;
$this->channel = $channel;
}

public function get_type() {
return $this->type;
}

public function get_name() {
return MySQL_Lexer::get_token_name( $this->type );
}

public function get_text() {
return $this->text;
}

public function get_channel() {
return $this->channel;
}

public function __toString() {
return $this->text . '<' . $this->type . ',' . $this->get_name() . '>';
}

public function extract_value() {
if ( MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->type ) {
return substr( $this->text, 1, -1 );
} elseif ( MySQL_Lexer::DOUBLE_QUOTED_TEXT === $this->type ) {
return substr( $this->text, 1, -1 );
} elseif ( MySQL_Lexer::SINGLE_QUOTED_TEXT === $this->type ) {
return substr( $this->text, 1, -1 );
} else {
return $this->text;
}
}
}
File renamed without changes.
Loading

0 comments on commit 17b2613

Please sign in to comment.