[2.0] More progress on the DQL parser. Added glimpse() method for the...

[2.0] More progress on the DQL parser. Added glimpse() method for the scanner/lexer that is equivalent to peek() immediately followed by resetPeek().

lib/Doctrine/ORM/Query/AST/LikeExpression.php

+<?php
+/* 
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+/**
+ * LikeExpression ::= StringExpression ["NOT"] "LIKE" string ["ESCAPE" char]
+ *
+ * @author robo
+ */
+class Doctrine_ORM_Query_AST_LikeExpression extends Doctrine_ORM_Query_AST
+{
+    private $_stringExpr;
+    private $_isNot;
+    private $_stringPattern;
+    private $_escapeChar;
+
+    public function __construct($stringExpr, $stringPattern, $isNot = false, $escapeChar = null)
+    {
+        $this->_stringExpr = $stringExpr;
+        $this->_stringPattern = $stringPattern;
+        $this->_isNot = $isNot;
+        $this->_escapeChar = $escapeChar;
+    }
+
+    public function isNot()
+    {
+        return $this->_isNot;
+    }
+
+    public function getStringExpression()
+    {
+        return $this->_stringExpr;
+    }
+
+    public function getStringPattern()
+    {
+        return $this->_stringPattern;
+    }
+    
+    public function getEscapeChar()
+    {
+        return $this->_escapeChar;
+    }
+}
+

lib/Doctrine/ORM/Query/Parser.php

@@ -232,9 +232,10 @@ class Doctrine_ORM_Query_Parser
     {
         // Parse & build AST
         $AST = $this->_QueryLanguage();
-
+        
         // Check for end of string
         if ($this->lookahead !== null) {
+            var_dump($this->lookahead);
             $this->syntaxError('end of string');
         }
 
@@ -995,10 +996,29 @@ class Doctrine_ORM_Query_Parser
     {
         $condPrimary = new Doctrine_ORM_Query_AST_ConditionalPrimary;
         if ($this->_isNextToken('(')) {
-            $this->match('(');
-            $conditionalExpression = $this->_ConditionalExpression();
-            $this->match(')');
-            $condPrimary->setConditionalExpression($conditionalExpression);
+            $numUnmatched = 1;
+            $peek = $this->_scanner->peek();
+            while ($numUnmatched > 0) {
+                if ($peek['value'] == ')') {
+                    --$numUnmatched;
+                } else if ($peek['value'] == '(') {
+                    ++$numUnmatched;
+                }
+                $peek = $this->_scanner->peek();
+            }
+            $this->_scanner->resetPeek();
+
+            //TODO: This is not complete, what about LIKE/BETWEEN/...etc?
+            $comparisonOps = array("=",  "<", "<=", "<>", ">", ">=", "!=");
+
+            if (in_array($peek['value'], $comparisonOps)) {
+                $condPrimary->setSimpleConditionalExpression($this->_SimpleConditionalExpression());
+            } else {
+                $this->match('(');
+                $conditionalExpression = $this->_ConditionalExpression();
+                $this->match(')');
+                $condPrimary->setConditionalExpression($conditionalExpression);
+            }
         } else {
             $condPrimary->setSimpleConditionalExpression($this->_SimpleConditionalExpression());
         }
@@ -1014,8 +1034,7 @@ class Doctrine_ORM_Query_Parser
     private function _SimpleConditionalExpression()
     {
         if ($this->_isNextToken(Doctrine_ORM_Query_Token::T_NOT)) {
-            $token = $this->_scanner->peek();
-            $this->_scanner->resetPeek();
+            $token = $this->_scanner->glimpse();
         } else {
             $token = $this->lookahead;
         }
@@ -1051,6 +1070,8 @@ class Doctrine_ORM_Query_Parser
                 default:
                     $this->syntaxError();
             }
+        } else if ($token['value'] == '(') {
+            return $this->_ComparisonExpression();
         } else {
             switch ($token['type']) {
                 case Doctrine_ORM_Query_Token::T_INTEGER:
@@ -1109,6 +1130,12 @@ class Doctrine_ORM_Query_Parser
         $terms = array();
         $terms[] = $this->_ArithmeticTerm();
         while ($this->lookahead['value'] == '+' || $this->lookahead['value'] == '-') {
+            if ($this->lookahead['value'] == '+') {
+                $this->match('+');
+            } else {
+                $this->match('-');
+            }
+            $terms[] = $this->token['value'];
             $terms[] = $this->_ArithmeticTerm();
         }
         return new Doctrine_ORM_Query_AST_SimpleArithmeticExpression($terms);
@@ -1122,6 +1149,12 @@ class Doctrine_ORM_Query_Parser
         $factors = array();
         $factors[] = $this->_ArithmeticFactor();
         while ($this->lookahead['value'] == '*' || $this->lookahead['value'] == '/') {
+            if ($this->lookahead['value'] == '*') {
+                $this->match('*');
+            } else {
+                $this->match('/');
+            }
+            $factors[] = $this->token['value'];
             $factors[] = $this->_ArithmeticFactor();
         }
         return new Doctrine_ORM_Query_AST_ArithmeticTerm($factors);
@@ -1148,16 +1181,27 @@ class Doctrine_ORM_Query_Parser
      */
     private function _ArithmeticPrimary()
     {
-        if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_IDENTIFIER) {
-            return $this->_StateFieldPathExpression();
-        }
         if ($this->lookahead['value'] === '(') {
-            return $this->_SimpleArithmeticExpression();
+            $this->match('(');
+            $expr = $this->_SimpleArithmeticExpression();
+            $this->match(')');
+            return $expr;
         }
-        if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_INPUT_PARAMETER) {
-            $this->match($this->lookahead['value']);
-            return new Doctrine_ORM_Query_AST_InputParameter($this->token['value']);
+        switch ($this->lookahead['type']) {
+            case Doctrine_ORM_Query_Token::T_IDENTIFIER:
+                return $this->_StateFieldPathExpression();
+            case Doctrine_ORM_Query_Token::T_INPUT_PARAMETER:
+                $this->match($this->lookahead['value']);
+                return new Doctrine_ORM_Query_AST_InputParameter($this->token['value']);
+            case Doctrine_ORM_Query_Token::T_STRING:
+            case Doctrine_ORM_Query_Token::T_INTEGER:
+            case Doctrine_ORM_Query_Token::T_FLOAT:
+                $this->match($this->lookahead['value']);
+                return $this->token['value'];
+            default:
+                $this->syntaxError();
         }
+        throw new Doctrine_Exception("Not yet implemented.");
         //TODO...
     }
 
@@ -1194,8 +1238,71 @@ class Doctrine_ORM_Query_Parser
                 $this->match('=');
                 return '<>';
             default:
-                $this->_parser->syntaxError('=, <, <=, <>, >, >=, !=');
+                $this->syntaxError('=, <, <=, <>, >, >=, !=');
                 break;
         }
     }
+
+    /**
+     * LikeExpression ::= StringExpression ["NOT"] "LIKE" string ["ESCAPE" char]
+     */
+    private function _LikeExpression()
+    {
+        $stringExpr = $this->_StringExpression();
+        $isNot = false;
+        if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_NOT) {
+            $this->match(Doctrine_ORM_Query_Token::T_NOT);
+            $isNot = true;
+        }
+        $this->match(Doctrine_ORM_Query_Token::T_LIKE);
+        $this->match(Doctrine_ORM_Query_Token::T_STRING);
+        $stringPattern = $this->token['value'];
+        $escapeChar = null;
+        if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_ESCAPE) {
+            $this->match(Doctrine_ORM_Query_Token::T_ESCAPE);
+            var_dump($this->lookahead);
+            //$this->match(Doctrine_ORM_Query_Token::T_)
+            //$escapeChar =
+        }
+        return new Doctrine_ORM_Query_AST_LikeExpression($stringExpr, $stringPattern, $isNot, $escapeChar);
+    }
+
+    /**
+     * StringExpression ::= StringPrimary | "(" Subselect ")"
+     */
+    private function _StringExpression()
+    {
+        if ($this->lookahead['value'] === '(') {
+            $peek = $this->_scanner->peek();
+            $this->_scanner->resetPeek();
+            if ($peek['type'] === Doctrine_ORM_Query_Token::T_SELECT) {
+                return $this->_Subselect();
+            }
+        }
+        return $this->_StringPrimary();
+    }
+
+    /**
+     * StringPrimary ::= StateFieldPathExpression | string | InputParameter | FunctionsReturningStrings | AggregateExpression
+     */
+    private function _StringPrimary()
+    {
+        if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_IDENTIFIER) {
+            $peek = $this->_scanner->peek();
+            $this->_scanner->resetPeek();
+            if ($peek['value'] == '.') {
+                return $this->_StateFieldPathExpression();
+            } else if ($peek['value'] == '(') {
+                //TODO... FunctionsReturningStrings or AggregateExpression
+            } else {
+                $this->syntaxError("'.' or '('");
+            }
+        } else if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_STRING) {
+            //TODO...
+        } else if ($this->lookahead['type'] === Doctrine_ORM_Query_Token::T_INPUT_PARAMETER) {
+            //TODO...
+        } else {
+            $this->syntaxError('StateFieldPathExpression | string | InputParameter | FunctionsReturningStrings | AggregateExpression');
+        }
+    }
 }

lib/Doctrine/ORM/Query/Scanner.php

@@ -91,7 +91,7 @@ class Doctrine_ORM_Query_Scanner
                 '[a-z_][a-z0-9_]*',
                 '(?:[0-9]+(?:[,\.][0-9]+)*)(?:e[+-]?[0-9]+)?',
                 "'(?:[^']|'')*'",
-                '\?[0-9]+|:[a-z]+'
+                '\?[0-9]+|:[a-z][a-z0-9_]+'
             );
             $regex = '/(' . implode(')|(', $patterns) . ')|\s+|(.)/i';
         }
@@ -179,7 +179,9 @@ class Doctrine_ORM_Query_Scanner
     }
 
     /**
-     * @todo Doc
+     * Moves the lookahead token forward.
+     *
+     * @return array|null The next token or NULL if there are no more tokens ahead.
      */
     public function peek()
     {
@@ -190,6 +192,18 @@ class Doctrine_ORM_Query_Scanner
         }
     }
 
+    /**
+     * Peeks at the next token, returns it and immediately resets the peek.
+     *
+     * @return array|null The next token or NULL if there are no more tokens ahead.
+     */
+    public function glimpse()
+    {
+        $peek = $this->peek();
+        $this->_peek = 0;
+        return $peek;
+    }
+
     /**
      * @todo Doc
      */
@@ -212,7 +226,6 @@ class Doctrine_ORM_Query_Scanner
     public function next()
     {
         $this->_peek = 0;
-
         if (isset($this->_tokens[$this->_position])) {
             return $this->_tokens[$this->_position++];
         } else {

lib/Doctrine/ORM/Query/SqlWalker.php

@@ -153,7 +153,7 @@ class Doctrine_ORM_Query_SqlWalker
                 $sql .= $sqlTableAlias . '.' . $class->getColumnName($fieldName) .
                         ' AS ' . $sqlTableAlias . '__' . $class->getColumnName($fieldName);
             } else if ($pathExpression->isSimpleStateFieldAssociationPathExpression()) {
-                echo "HERE!!";
+                throw new Doctrine_Exception("Not yet implemented.");
             } else {
                 throw new Doctrine_ORM_Query_Exception("Encountered invalid PathExpression during SQL construction.");
             }
@@ -175,6 +175,7 @@ class Doctrine_ORM_Query_SqlWalker
             $columnName = $qComp['metadata']->getColumnName($fieldName);
             
             $sql .= $aggExpr->getFunctionName() . '(';
+            if ($aggExpr->isDistinct()) $sql .= 'DISTINCT ';
             $sql .= $this->_dqlToSqlAliasMap[$dqlAlias] . '.' . $columnName;
             $sql .= ') AS dctrn__' . $alias;
         }
@@ -229,18 +230,16 @@ class Doctrine_ORM_Query_SqlWalker
     {
         $sql = ' WHERE ';
         $condExpr = $whereClause->getConditionalExpression();
-        foreach ($condExpr->getConditionalTerms() as $term) {
-            $sql .= $this->walkConditionalTerm($term);
-        }
+        $sql .= implode(' OR ', array_map(array(&$this, 'walkConditionalTerm'),
+                $condExpr->getConditionalTerms()));
         return $sql;
     }
 
     public function walkConditionalTerm($condTerm)
     {
         $sql = '';
-        foreach ($condTerm->getConditionalFactors() as $factor) {
-            $sql .= $this->walkConditionalFactor($factor);
-        }
+        $sql .= implode(' AND ', array_map(array(&$this, 'walkConditionalFactor'),
+                $condTerm->getConditionalFactors()));
         return $sql;
     }
 
@@ -254,9 +253,25 @@ class Doctrine_ORM_Query_SqlWalker
             if ($simpleCond instanceof Doctrine_ORM_Query_AST_ComparisonExpression) {
                 $sql .= $this->walkComparisonExpression($simpleCond);
             }
+            else if ($simpleCond instanceof Doctrine_ORM_Query_AST_LikeExpression) {
+                $sql .= $this->walkLikeExpression($simpleCond);
+            }
             // else if ...
+        } else if ($primary->isConditionalExpression()) {
+            $sql .= '(' . implode(' OR ', array_map(array(&$this, 'walkConditionalTerm'),
+                    $primary->getConditionalExpression()->getConditionalTerms())) . ')';
         }
+        return $sql;
+    }
 
+    public function walkLikeExpression($likeExpr)
+    {
+        $sql = '';
+        $stringExpr = $likeExpr->getStringExpression();
+        if ($stringExpr instanceof Doctrine_ORM_Query_AST_PathExpression) {
+            $sql .= $this->walkPathExpression($stringExpr);
+        } //TODO else...
+        $sql .= ' LIKE ' . $likeExpr->getStringPattern();
         return $sql;
     }
 
@@ -288,18 +303,19 @@ class Doctrine_ORM_Query_SqlWalker
 
     public function walkArithmeticTerm($term)
     {
-        $sql = '';
-        foreach ($term->getArithmeticFactors() as $factor) {
-            $sql .= $this->walkArithmeticFactor($factor);
-        }
-        return $sql;
+        if (is_string($term)) return $term;
+        return implode(' ', array_map(array(&$this, 'walkArithmeticFactor'),
+                $term->getArithmeticFactors()));
     }
 
     public function walkArithmeticFactor($factor)
     {
+        if (is_string($factor)) return $factor;
         $sql = '';
         $primary = $factor->getArithmeticPrimary();
-        if ($primary instanceof Doctrine_ORM_Query_AST_PathExpression) {
+        if (is_numeric($primary)) {
+            $sql .= $primary;
+        } else if ($primary instanceof Doctrine_ORM_Query_AST_PathExpression) {
             $sql .= $this->walkPathExpression($primary);
         } else if ($primary instanceof Doctrine_ORM_Query_AST_InputParameter) {
             if ($primary->isNamed()) {
@@ -307,6 +323,8 @@ class Doctrine_ORM_Query_SqlWalker
             } else {
                 $sql .= '?';
             }
+        } else if ($primary instanceof Doctrine_ORM_Query_AST_SimpleArithmeticExpression) {
+            $sql .= '(' . $this->walkSimpleArithmeticExpression($primary) . ')';
         }
          
         // else...
@@ -314,6 +332,12 @@ class Doctrine_ORM_Query_SqlWalker
         return $sql;
     }
 
+    public function walkSimpleArithmeticExpression($simpleArithmeticExpr)
+    {
+        return implode(' ', array_map(array(&$this, 'walkArithmeticTerm'),
+                $simpleArithmeticExpr->getArithmeticTerms()));
+    }
+
     public function walkPathExpression($pathExpr)
     {
         $sql = '';

query-language.txt

@@ -239,7 +239,7 @@ ComparisonExpression     ::= ArithmeticExpression ComparisonOperator ( Quantifie
                              DatetimeExpression ComparisonOperator (DatetimeExpression | QuantifiedExpression) |
                              EntityExpression ("=" | "<>") (EntityExpression | QuantifiedExpression)
 InExpression             ::= StateFieldPathExpression ["NOT"] "IN" "(" (Literal {"," Literal}* | Subselect) ")"
-LikeExpression           ::= ["NOT"] "LIKE" string ["ESCAPE" char]
+LikeExpression           ::= StringExpression ["NOT"] "LIKE" string ["ESCAPE" char]
 NullComparisonExpression ::= (SingleValuedPathExpression | InputParameter) "IS" ["NOT"] "NULL"
 ExistsExpression         ::= ["NOT"] "EXISTS" "(" Subselect ")"
 ComparisonOperator       ::= "=" | "<" | "<=" | "<>" | ">" | ">=" | "!="

tests/Orm/Query/SelectSqlGenerationTest.php

@@ -110,7 +110,7 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase
         );
     }
 
-    public function testWhereClauseInSelect()
+    public function testWhereClauseInSelectWithPositionalParameter()
     {
         $this->assertSqlGeneration(
             'select u from ForumUser u where u.id = ?1',
@@ -118,36 +118,88 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase
         );
     }
 
-/*    public function testAggregateFunctionWithDistinctInSelect()
+    public function testWhereClauseInSelectWithNamedParameter()
     {
         $this->assertSqlGeneration(
-            'SELECT COUNT(DISTINCT u.name) FROM CmsUser u',
-            'SELECT COUNT(DISTINCT cu.name) AS dctrn__0 FROM cms_user cu WHERE 1 = 1'
+            'select u from ForumUser u where u.username = :name',
+            'SELECT fu.id AS fu__id, fu.username AS fu__username FROM ForumUser fu WHERE fu.username = :name'
         );
     }
 
+    public function testWhereANDClauseInSelectWithNamedParameter()
+    {
+        $this->assertSqlGeneration(
+            'select u from ForumUser u where u.username = :name and u.username = :name2',
+            'SELECT fu.id AS fu__id, fu.username AS fu__username FROM ForumUser fu WHERE fu.username = :name AND fu.username = :name2'
+        );
+    }
 
-    public function testFunctionalExpressionsSupportedInWherePart()
+    public function testCombinedWhereClauseInSelectWithNamedParameter()
     {
         $this->assertSqlGeneration(
-            "SELECT u.name FROM CmsUser u WHERE TRIM(u.name) = 'someone'",
-            // String quoting in the SQL usually depends on the database platform.
-            // This test works with a mock connection which uses ' for string quoting.
-            "SELECT cu.name AS cu__name FROM cms_user cu WHERE TRIM(cu.name) = 'someone'"
+            'select u from ForumUser u where (u.username = :name OR u.username = :name2) AND u.id = :id',
+            'SELECT fu.id AS fu__id, fu.username AS fu__username FROM ForumUser fu WHERE (fu.username = :name OR fu.username = :name2) AND fu.id = :id'
         );
     }
 
+    public function testAggregateFunctionWithDistinctInSelect()
+    {
+        $this->assertSqlGeneration(
+            'SELECT COUNT(DISTINCT u.name) FROM CmsUser u',
+            'SELECT COUNT(DISTINCT cu.name) AS dctrn__0 FROM CmsUser cu'
+        );
+    }
 
     // Ticket #668
     public function testKeywordUsageInStringParam()
     {
         $this->assertSqlGeneration(
             "SELECT u.name FROM CmsUser u WHERE u.name LIKE '%foo OR bar%'",
-            "SELECT cu.name AS cu__name FROM cms_user cu WHERE cu.name LIKE '%foo OR bar%'"
+            "SELECT cu.name AS cu__name FROM CmsUser cu WHERE cu.name LIKE '%foo OR bar%'"
         );
     }
 
+    public function testArithmeticExpressionsSupportedInWherePart()
+    {
+        $this->assertSqlGeneration(
+            'SELECT u FROM CmsUser u WHERE ((u.id + 5000) * u.id + 3) < 10000000',
+            'SELECT cu.id AS cu__id, cu.status AS cu__status, cu.username AS cu__username, cu.name AS cu__name FROM CmsUser cu WHERE ((cu.id + 5000) * cu.id + 3) < 10000000'
+        );
+    }
+
+    public function testPlainJoinWithoutClause()
+    {
+        $this->assertSqlGeneration(
+            'SELECT u.id, a.id from CmsUser u LEFT JOIN u.articles a',
+            'SELECT cu.id AS cu__id, ca.id AS ca__id FROM CmsUser cu LEFT JOIN CmsArticle ca ON cu.id = ca.user_id'
+        );
+        $this->assertSqlGeneration(
+            'SELECT u.id, a.id from CmsUser u JOIN u.articles a',
+            'SELECT cu.id AS cu__id, ca.id AS ca__id FROM CmsUser cu INNER JOIN CmsArticle ca ON cu.id = ca.user_id'
+        );
+    }
 
+    public function testDeepJoin()
+    {
+        $this->assertSqlGeneration(
+            'SELECT u.id, a.id, p, c.id from CmsUser u JOIN u.articles a JOIN u.phonenumbers p JOIN a.comments c',
+            'SELECT cu.id AS cu__id, ca.id AS ca__id, cp.phonenumber AS cp__phonenumber, cc.id AS cc__id FROM CmsUser cu INNER JOIN CmsArticle ca ON cu.id = ca.user_id INNER JOIN CmsPhonenumber cp ON cu.id = cp.user_id INNER JOIN CmsComment cc ON ca.id = cc.article_id'
+        );
+    }
+
+/*
+    public function testFunctionalExpressionsSupportedInWherePart()
+    {
+        $this->assertSqlGeneration(
+            "SELECT u.name FROM CmsUser u WHERE TRIM(u.name) = 'someone'",
+            // String quoting in the SQL usually depends on the database platform.
+            // This test works with a mock connection which uses ' for string quoting.
+            "SELECT cu.name AS cu__name FROM cms_user cu WHERE TRIM(cu.name) = 'someone'"
+        );
+    }
+*/
+
+/*
     // Ticket #973
     public function testSingleInValueWithoutSpace()
     {
@@ -168,15 +220,6 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase
     }
 
 
-    public function testArithmeticExpressionsSupportedInWherePart()
-    {
-        $this->assertSqlGeneration(
-            'SELECT u.* FROM CmsUser u WHERE ((u.id + 5000) * u.id + 3) < 10000000',
-            'SELECT cu.id AS cu__id, cu.status AS cu__status, cu.username AS cu__username, cu.name AS cu__name FROM cms_user cu WHERE ((cu.id + 5000) * cu.id + 3) < 10000000'
-        );
-    }
-
-
     public function testInExpressionSupportedInWherePart()
     {
         $this->assertSqlGeneration(
@@ -194,17 +237,5 @@ class Orm_Query_SelectSqlGenerationTest extends Doctrine_OrmTestCase
         );
     }
 
-
-    public function testPlainJoinWithoutClause()
-    {
-        $this->assertSqlGeneration(
-            'SELECT u.id, a.id from CmsUser u LEFT JOIN u.articles a',
-            'SELECT cu.id AS cu__id, ca.id AS ca__id FROM cms_user cu LEFT JOIN cms_article ca ON cu.id = ca.user_id WHERE 1 = 1'
-        );
-        $this->assertSqlGeneration(
-            'SELECT u.id, a.id from CmsUser u JOIN u.articles a',
-            'SELECT cu.id AS cu__id, ca.id AS ca__id FROM cms_user cu INNER JOIN cms_article ca ON cu.id = ca.user_id WHERE 1 = 1'
-        );
-    }
 */
 }
\ No newline at end of file