From 4ab2ba7dcbfca13ad93e5a86d63a20a7a6a24f55 Mon Sep 17 00:00:00 2001 From: romanb Date: Wed, 21 Jan 2009 18:25:05 +0000 Subject: [PATCH] [2.0] More progress on the DQL parser. Added glimpse() method for the scanner/lexer that is equivalent to peek() immediately followed by resetPeek(). --- lib/Doctrine/ORM/Query/AST/LikeExpression.php | 47 ++++++ lib/Doctrine/ORM/Query/Parser.php | 137 ++++++++++++++++-- lib/Doctrine/ORM/Query/Scanner.php | 19 ++- lib/Doctrine/ORM/Query/SqlWalker.php | 50 +++++-- query-language.txt | 2 +- tests/Orm/Query/SelectSqlGenerationTest.php | 103 ++++++++----- 6 files changed, 290 insertions(+), 68 deletions(-) create mode 100644 lib/Doctrine/ORM/Query/AST/LikeExpression.php diff --git a/lib/Doctrine/ORM/Query/AST/LikeExpression.php b/lib/Doctrine/ORM/Query/AST/LikeExpression.php new file mode 100644 index 000000000..c2848e4ab --- /dev/null +++ b/lib/Doctrine/ORM/Query/AST/LikeExpression.php @@ -0,0 +1,47 @@ +_stringExpr = $stringExpr; + $this->_stringPattern = $stringPattern; + $this->_isNot = $isNot; + $this->_escapeChar = $escapeChar; + } + + public function isNot() + { + return $this->_isNot; + } + + public function getStringExpression() + { + return $this->_stringExpr; + } + + public function getStringPattern() + { + return $this->_stringPattern; + } + + public function getEscapeChar() + { + return $this->_escapeChar; + } +} + diff --git a/lib/Doctrine/ORM/Query/Parser.php b/lib/Doctrine/ORM/Query/Parser.php index 105ceedff..1fada861c 100644 --- a/lib/Doctrine/ORM/Query/Parser.php +++ b/lib/Doctrine/ORM/Query/Parser.php @@ -232,9 +232,10 @@ class Doctrine_ORM_Query_Parser { // Parse & build AST $AST = $this->_QueryLanguage(); - + // Check for end of string if ($this->lookahead !== null) { + var_dump($this->lookahead); $this->syntaxError('end of string'); } @@ -995,10 +996,29 @@ class Doctrine_ORM_Query_Parser { $condPrimary = new Doctrine_ORM_Query_AST_ConditionalPrimary; if ($this->_isNextToken('(')) { - $this->match('('); - $conditionalExpression = $this->_ConditionalExpression(); - $this->match(')'); - $condPrimary->setConditionalExpression($conditionalExpression); + $numUnmatched = 1; + $peek = $this->_scanner->peek(); + while ($numUnmatched > 0) { + if ($peek['value'] == ')') { + --$numUnmatched; + } else if ($peek['value'] == '(') { + ++$numUnmatched; + } + $peek = $this->_scanner->peek(); + } + $this->_scanner->resetPeek(); + + //TODO: This is not complete, what about LIKE/BETWEEN/...etc? + $comparisonOps = array("=", "<", "<=", "<>", ">", ">=", "!="); + + if (in_array($peek['value'], $comparisonOps)) { + $condPrimary->setSimpleConditionalExpression($this->_SimpleConditionalExpression()); + } else { + $this->match('('); + $conditionalExpression = $this->_ConditionalExpression(); + $this->match(')'); + $condPrimary->setConditionalExpression($conditionalExpression); + } } else { $condPrimary->setSimpleConditionalExpression($this->_SimpleConditionalExpression()); } @@ -1014,8 +1034,7 @@ class Doctrine_ORM_Query_Parser private function _SimpleConditionalExpression() { if ($this->_isNextToken(Doctrine_ORM_Query_Token::T_NOT)) { - $token = $this->_scanner->peek(); - $this->_scanner->resetPeek(); + $token = $this->_scanner->glimpse(); } else { $token = $this->lookahead; } @@ -1051,6 +1070,8 @@ class Doctrine_ORM_Query_Parser default: $this->syntaxError(); } + } else if ($token['value'] == '(') { + return $this->_ComparisonExpression(); } else { switch ($token['type']) { case Doctrine_ORM_Query_Token::T_INTEGER: @@ -1109,6 +1130,12 @@ class Doctrine_ORM_Query_Parser $terms = array(); $terms[] = $this->_ArithmeticTerm(); while ($this->lookahead['value'] == '+' || $this->lookahead['value'] == '-') { + if ($this->lookahead['value'] == '+') { + $this->match('+'); + } else { + $this->match('-'); + } + $terms[] = $this->token['value']; $terms[] = $this->_ArithmeticTerm(); } return new Doctrine_ORM_Query_AST_SimpleArithmeticExpression($terms); @@ -1122,6 +1149,12 @@ class Doctrine_ORM_Query_Parser $factors = array(); $factors[] = $this->_ArithmeticFactor(); while ($this->lookahead['value'] == '*' || $this->lookahead['value'] == '/') { + if ($this->lookahead['value'] == '*') { + $this->match('*'); + } else { + $this->match('/'); + } + $factors[] = $this->token['value']; $factors[] = $this->_ArithmeticFactor(); } return new Doctrine_ORM_Query_AST_ArithmeticTerm($factors); @@ -1148,16 +1181,27 @@ class Doctrine_ORM_Query_Parser */ private function _ArithmeticPrimary() { - if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_IDENTIFIER) { - return $this->_StateFieldPathExpression(); - } if ($this->lookahead['value'] === '(') { - return $this->_SimpleArithmeticExpression(); + $this->match('('); + $expr = $this->_SimpleArithmeticExpression(); + $this->match(')'); + return $expr; } - if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_INPUT_PARAMETER) { - $this->match($this->lookahead['value']); - return new Doctrine_ORM_Query_AST_InputParameter($this->token['value']); + switch ($this->lookahead['type']) { + case Doctrine_ORM_Query_Token::T_IDENTIFIER: + return $this->_StateFieldPathExpression(); + case Doctrine_ORM_Query_Token::T_INPUT_PARAMETER: + $this->match($this->lookahead['value']); + return new Doctrine_ORM_Query_AST_InputParameter($this->token['value']); + case Doctrine_ORM_Query_Token::T_STRING: + case Doctrine_ORM_Query_Token::T_INTEGER: + case Doctrine_ORM_Query_Token::T_FLOAT: + $this->match($this->lookahead['value']); + return $this->token['value']; + default: + $this->syntaxError(); } + throw new Doctrine_Exception("Not yet implemented."); //TODO... } @@ -1194,8 +1238,71 @@ class Doctrine_ORM_Query_Parser $this->match('='); return '<>'; default: - $this->_parser->syntaxError('=, <, <=, <>, >, >=, !='); + $this->syntaxError('=, <, <=, <>, >, >=, !='); break; } } + + /** + * LikeExpression ::= StringExpression ["NOT"] "LIKE" string ["ESCAPE" char] + */ + private function _LikeExpression() + { + $stringExpr = $this->_StringExpression(); + $isNot = false; + if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_NOT) { + $this->match(Doctrine_ORM_Query_Token::T_NOT); + $isNot = true; + } + $this->match(Doctrine_ORM_Query_Token::T_LIKE); + $this->match(Doctrine_ORM_Query_Token::T_STRING); + $stringPattern = $this->token['value']; + $escapeChar = null; + if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_ESCAPE) { + $this->match(Doctrine_ORM_Query_Token::T_ESCAPE); + var_dump($this->lookahead); + //$this->match(Doctrine_ORM_Query_Token::T_) + //$escapeChar = + } + return new Doctrine_ORM_Query_AST_LikeExpression($stringExpr, $stringPattern, $isNot, $escapeChar); + } + + /** + * StringExpression ::= StringPrimary | "(" Subselect ")" + */ + private function _StringExpression() + { + if ($this->lookahead['value'] === '(') { + $peek = $this->_scanner->peek(); + $this->_scanner->resetPeek(); + if ($peek['type'] === Doctrine_ORM_Query_Token::T_SELECT) { + return $this->_Subselect(); + } + } + return $this->_StringPrimary(); + } + + /** + * StringPrimary ::= StateFieldPathExpression | string | InputParameter | FunctionsReturningStrings | AggregateExpression + */ + private function _StringPrimary() + { + if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_IDENTIFIER) { + $peek = $this->_scanner->peek(); + $this->_scanner->resetPeek(); + if ($peek['value'] == '.') { + return $this->_StateFieldPathExpression(); + } else if ($peek['value'] == '(') { + //TODO... FunctionsReturningStrings or AggregateExpression + } else { + $this->syntaxError("'.' or '('"); + } + } else if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_STRING) { + //TODO... + } else if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_INPUT_PARAMETER) { + //TODO... + } else { + $this->syntaxError('StateFieldPathExpression | string | InputParameter | FunctionsReturningStrings | AggregateExpression'); + } + } } diff --git a/lib/Doctrine/ORM/Query/Scanner.php b/lib/Doctrine/ORM/Query/Scanner.php index a0316473e..6c79f39d8 100644 --- a/lib/Doctrine/ORM/Query/Scanner.php +++ b/lib/Doctrine/ORM/Query/Scanner.php @@ -91,7 +91,7 @@ class Doctrine_ORM_Query_Scanner '[a-z_][a-z0-9_]*', '(?:[0-9]+(?:[,\.][0-9]+)*)(?:e[+-]?[0-9]+)?', "'(?:[^']|'')*'", - '\?[0-9]+|:[a-z]+' + '\?[0-9]+|:[a-z][a-z0-9_]+' ); $regex = '/(' . implode(')|(', $patterns) . ')|\s+|(.)/i'; } @@ -179,7 +179,9 @@ class Doctrine_ORM_Query_Scanner } /** - * @todo Doc + * Moves the lookahead token forward. + * + * @return array|null The next token or NULL if there are no more tokens ahead. */ public function peek() { @@ -190,6 +192,18 @@ class Doctrine_ORM_Query_Scanner } } + /** + * Peeks at the next token, returns it and immediately resets the peek. + * + * @return array|null The next token or NULL if there are no more tokens ahead. + */ + public function glimpse() + { + $peek = $this->peek(); + $this->_peek = 0; + return $peek; + } + /** * @todo Doc */ @@ -212,7 +226,6 @@ class Doctrine_ORM_Query_Scanner public function next() { $this->_peek = 0; - if (isset($this->_tokens[$this->_position])) { return $this->_tokens[$this->_position++]; } else { diff --git a/lib/Doctrine/ORM/Query/SqlWalker.php b/lib/Doctrine/ORM/Query/SqlWalker.php index d1c0eaa22..bd66432c9 100644 --- a/lib/Doctrine/ORM/Query/SqlWalker.php +++ b/lib/Doctrine/ORM/Query/SqlWalker.php @@ -153,7 +153,7 @@ class Doctrine_ORM_Query_SqlWalker $sql .= $sqlTableAlias . '.' . $class->getColumnName($fieldName) . ' AS ' . $sqlTableAlias . '__' . $class->getColumnName($fieldName); } else if ($pathExpression->isSimpleStateFieldAssociationPathExpression()) { - echo "HERE!!"; + throw new Doctrine_Exception("Not yet implemented."); } else { throw new Doctrine_ORM_Query_Exception("Encountered invalid PathExpression during SQL construction."); } @@ -175,6 +175,7 @@ class Doctrine_ORM_Query_SqlWalker $columnName = $qComp['metadata']->getColumnName($fieldName); $sql .= $aggExpr->getFunctionName() . '('; + if ($aggExpr->isDistinct()) $sql .= 'DISTINCT '; $sql .= $this->_dqlToSqlAliasMap[$dqlAlias] . '.' . $columnName; $sql .= ') AS dctrn__' . $alias; } @@ -229,18 +230,16 @@ class Doctrine_ORM_Query_SqlWalker { $sql = ' WHERE '; $condExpr = $whereClause->getConditionalExpression(); - foreach ($condExpr->getConditionalTerms() as $term) { - $sql .= $this->walkConditionalTerm($term); - } + $sql .= implode(' OR ', array_map(array(&$this, 'walkConditionalTerm'), + $condExpr->getConditionalTerms())); return $sql; } public function walkConditionalTerm($condTerm) { $sql = ''; - foreach ($condTerm->getConditionalFactors() as $factor) { - $sql .= $this->walkConditionalFactor($factor); - } + $sql .= implode(' AND ', array_map(array(&$this, 'walkConditionalFactor'), + $condTerm->getConditionalFactors())); return $sql; } @@ -254,9 +253,25 @@ class Doctrine_ORM_Query_SqlWalker if ($simpleCond instanceof Doctrine_ORM_Query_AST_ComparisonExpression) { $sql .= $this->walkComparisonExpression($simpleCond); } + else if ($simpleCond instanceof Doctrine_ORM_Query_AST_LikeExpression) { + $sql .= $this->walkLikeExpression($simpleCond); + } // else if ... + } else if ($primary->isConditionalExpression()) { + $sql .= '(' . implode(' OR ', array_map(array(&$this, 'walkConditionalTerm'), + $primary->getConditionalExpression()->getConditionalTerms())) . ')'; } + return $sql; + } + public function walkLikeExpression($likeExpr) + { + $sql = ''; + $stringExpr = $likeExpr->getStringExpression(); + if ($stringExpr instanceof Doctrine_ORM_Query_AST_PathExpression) { + $sql .= $this->walkPathExpression($stringExpr); + } //TODO else... + $sql .= ' LIKE ' . $likeExpr->getStringPattern(); return $sql; } @@ -288,18 +303,19 @@ class Doctrine_ORM_Query_SqlWalker public function walkArithmeticTerm($term) { - $sql = ''; - foreach ($term->getArithmeticFactors() as $factor) { - $sql .= $this->walkArithmeticFactor($factor); - } - return $sql; + if (is_string($term)) return $term; + return implode(' ', array_map(array(&$this, 'walkArithmeticFactor'), + $term->getArithmeticFactors())); } public function walkArithmeticFactor($factor) { + if (is_string($factor)) return $factor; $sql = ''; $primary = $factor->getArithmeticPrimary(); - if ($primary instanceof Doctrine_ORM_Query_AST_PathExpression) { + if (is_numeric($primary)) { + $sql .= $primary; + } else if ($primary instanceof Doctrine_ORM_Query_AST_PathExpression) { $sql .= $this->walkPathExpression($primary); } else if ($primary instanceof Doctrine_ORM_Query_AST_InputParameter) { if ($primary->isNamed()) { @@ -307,6 +323,8 @@ class Doctrine_ORM_Query_SqlWalker } else { $sql .= '?'; } + } else if ($primary instanceof Doctrine_ORM_Query_AST_SimpleArithmeticExpression) { + $sql .= '(' . $this->walkSimpleArithmeticExpression($primary) . ')'; } // else... @@ -314,6 +332,12 @@ class Doctrine_ORM_Query_SqlWalker return $sql; } + public function walkSimpleArithmeticExpression($simpleArithmeticExpr) + { + return implode(' ', array_map(array(&$this, 'walkArithmeticTerm'), + $simpleArithmeticExpr->getArithmeticTerms())); + } + public function walkPathExpression($pathExpr) { $sql = ''; diff --git a/query-language.txt b/query-language.txt index 0056cdce0..7955aadb5 100644 --- a/query-language.txt +++ b/query-language.txt @@ -239,7 +239,7 @@ ComparisonExpression ::= ArithmeticExpression ComparisonOperator ( Quantifie DatetimeExpression ComparisonOperator (DatetimeExpression | QuantifiedExpression) | EntityExpression ("=" | "<>") (EntityExpression | QuantifiedExpression) InExpression ::= StateFieldPathExpression ["NOT"] "IN" "(" (Literal {"," Literal}* | Subselect) ")" -LikeExpression ::= ["NOT"] "LIKE" string ["ESCAPE" char] +LikeExpression ::= StringExpression ["NOT"] "LIKE" string ["ESCAPE" char] NullComparisonExpression ::= (SingleValuedPathExpression | InputParameter) "IS" ["NOT"] "NULL" ExistsExpression ::= ["NOT"] "EXISTS" "(" Subselect ")" ComparisonOperator ::= "=" | "<" | "<=" | "<>" | ">" | ">=" | "!=" diff --git a/tests/Orm/Query/SelectSqlGenerationTest.php b/tests/Orm/Query/SelectSqlGenerationTest.php index 500e119ba..07e2e7093 100755 --- a/tests/Orm/Query/SelectSqlGenerationTest.php +++ b/tests/Orm/Query/SelectSqlGenerationTest.php @@ -110,7 +110,7 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase ); } - public function testWhereClauseInSelect() + public function testWhereClauseInSelectWithPositionalParameter() { $this->assertSqlGeneration( 'select u from ForumUser u where u.id = ?1', @@ -118,15 +118,76 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase ); } -/* public function testAggregateFunctionWithDistinctInSelect() + public function testWhereClauseInSelectWithNamedParameter() { $this->assertSqlGeneration( - 'SELECT COUNT(DISTINCT u.name) FROM CmsUser u', - 'SELECT COUNT(DISTINCT cu.name) AS dctrn__0 FROM cms_user cu WHERE 1 = 1' + 'select u from ForumUser u where u.username = :name', + 'SELECT fu.id AS fu__id, fu.username AS fu__username FROM ForumUser fu WHERE fu.username = :name' ); } + public function testWhereANDClauseInSelectWithNamedParameter() + { + $this->assertSqlGeneration( + 'select u from ForumUser u where u.username = :name and u.username = :name2', + 'SELECT fu.id AS fu__id, fu.username AS fu__username FROM ForumUser fu WHERE fu.username = :name AND fu.username = :name2' + ); + } + public function testCombinedWhereClauseInSelectWithNamedParameter() + { + $this->assertSqlGeneration( + 'select u from ForumUser u where (u.username = :name OR u.username = :name2) AND u.id = :id', + 'SELECT fu.id AS fu__id, fu.username AS fu__username FROM ForumUser fu WHERE (fu.username = :name OR fu.username = :name2) AND fu.id = :id' + ); + } + + public function testAggregateFunctionWithDistinctInSelect() + { + $this->assertSqlGeneration( + 'SELECT COUNT(DISTINCT u.name) FROM CmsUser u', + 'SELECT COUNT(DISTINCT cu.name) AS dctrn__0 FROM CmsUser cu' + ); + } + + // Ticket #668 + public function testKeywordUsageInStringParam() + { + $this->assertSqlGeneration( + "SELECT u.name FROM CmsUser u WHERE u.name LIKE '%foo OR bar%'", + "SELECT cu.name AS cu__name FROM CmsUser cu WHERE cu.name LIKE '%foo OR bar%'" + ); + } + + public function testArithmeticExpressionsSupportedInWherePart() + { + $this->assertSqlGeneration( + 'SELECT u FROM CmsUser u WHERE ((u.id + 5000) * u.id + 3) < 10000000', + 'SELECT cu.id AS cu__id, cu.status AS cu__status, cu.username AS cu__username, cu.name AS cu__name FROM CmsUser cu WHERE ((cu.id + 5000) * cu.id + 3) < 10000000' + ); + } + + public function testPlainJoinWithoutClause() + { + $this->assertSqlGeneration( + 'SELECT u.id, a.id from CmsUser u LEFT JOIN u.articles a', + 'SELECT cu.id AS cu__id, ca.id AS ca__id FROM CmsUser cu LEFT JOIN CmsArticle ca ON cu.id = ca.user_id' + ); + $this->assertSqlGeneration( + 'SELECT u.id, a.id from CmsUser u JOIN u.articles a', + 'SELECT cu.id AS cu__id, ca.id AS ca__id FROM CmsUser cu INNER JOIN CmsArticle ca ON cu.id = ca.user_id' + ); + } + + public function testDeepJoin() + { + $this->assertSqlGeneration( + 'SELECT u.id, a.id, p, c.id from CmsUser u JOIN u.articles a JOIN u.phonenumbers p JOIN a.comments c', + 'SELECT cu.id AS cu__id, ca.id AS ca__id, cp.phonenumber AS cp__phonenumber, cc.id AS cc__id FROM CmsUser cu INNER JOIN CmsArticle ca ON cu.id = ca.user_id INNER JOIN CmsPhonenumber cp ON cu.id = cp.user_id INNER JOIN CmsComment cc ON ca.id = cc.article_id' + ); + } + +/* public function testFunctionalExpressionsSupportedInWherePart() { $this->assertSqlGeneration( @@ -136,18 +197,9 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase "SELECT cu.name AS cu__name FROM cms_user cu WHERE TRIM(cu.name) = 'someone'" ); } +*/ - - // Ticket #668 - public function testKeywordUsageInStringParam() - { - $this->assertSqlGeneration( - "SELECT u.name FROM CmsUser u WHERE u.name LIKE '%foo OR bar%'", - "SELECT cu.name AS cu__name FROM cms_user cu WHERE cu.name LIKE '%foo OR bar%'" - ); - } - - +/* // Ticket #973 public function testSingleInValueWithoutSpace() { @@ -168,15 +220,6 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase } - public function testArithmeticExpressionsSupportedInWherePart() - { - $this->assertSqlGeneration( - 'SELECT u.* FROM CmsUser u WHERE ((u.id + 5000) * u.id + 3) < 10000000', - 'SELECT cu.id AS cu__id, cu.status AS cu__status, cu.username AS cu__username, cu.name AS cu__name FROM cms_user cu WHERE ((cu.id + 5000) * cu.id + 3) < 10000000' - ); - } - - public function testInExpressionSupportedInWherePart() { $this->assertSqlGeneration( @@ -194,17 +237,5 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase ); } - - public function testPlainJoinWithoutClause() - { - $this->assertSqlGeneration( - 'SELECT u.id, a.id from CmsUser u LEFT JOIN u.articles a', - 'SELECT cu.id AS cu__id, ca.id AS ca__id FROM cms_user cu LEFT JOIN cms_article ca ON cu.id = ca.user_id WHERE 1 = 1' - ); - $this->assertSqlGeneration( - 'SELECT u.id, a.id from CmsUser u JOIN u.articles a', - 'SELECT cu.id AS cu__id, ca.id AS ca__id FROM cms_user cu INNER JOIN cms_article ca ON cu.id = ca.user_id WHERE 1 = 1' - ); - } */ } \ No newline at end of file