updates to the lexer of the new dql parser

draft/Doctrine/Query/Production/AggregateExpression.php

+<?php
+/*
+ *  $Id$
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This software consists of voluntary contributions made by many individuals
+ * and is licensed under the LGPL. For more information, see
+ * <http://www.phpdoctrine.org>.
+ */
+
+/**
+ * AggregateExpression =
+ *     ("AVG" | "MAX" | "MIN" | "SUM" | "COUNT") "(" ["DISTINCT"] Expression ")"
+ *
+ * @package     Doctrine
+ * @subpackage  Query
+ * @author      Janne Vanhala <jpvanhal@cc.hut.fi>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        http://www.phpdoctrine.org
+ * @since       1.0
+ * @version     $Revision$
+ */
+class Doctrine_Query_Production_AggregateExpression extends Doctrine_Query_Production
+{
+    public function execute(array $params = array())
+    {
+        $token = $this->_parser->lookahead;
+
+        switch ($token['type']) {
+            case Doctrine_Query_Token::T_AVG:
+            case Doctrine_Query_Token::T_MAX:
+            case Doctrine_Query_Token::T_MIN:
+            case Doctrine_Query_Token::T_SUM:
+            case Doctrine_Query_Token::T_COUNT:
+                $this->_parser->match($token['type']);
+            break;
+
+            default:
+                $this->_parser->logError();
+        }
+
+        $this->_parser->match('(');
+
+        if ($this->_isNextToken(Doctrine_Query_Token::T_DISTINCT)) {
+            $this->_parser->match(Doctrine_Query_Token::T_DISTINCT);
+        }
+
+        $this->Expression();
+
+        $this->_parser->match(')');
+    }
+}

draft/Doctrine/Query/Production/ComparisonExpression.php

@@ -32,6 +32,14 @@
  */
 class Doctrine_Query_Production_ComparisonExpression extends Doctrine_Query_Production
 {
+    private function _isSubquery()
+    {
+        $lookahead = $this->_parser->lookahead;
+        $next = $this->_parser->getScanner()->peek();
+
+        return $lookahead['value'] === '(' && $next['type'] === Doctrine_Query_Token::T_SELECT;
+    }
+
     public function execute(array $params = array())
     {
         $this->ComparisonOperator();

draft/Doctrine/Query/Production/ComparisonOperator.php

@@ -53,7 +53,7 @@ class Doctrine_Query_Production_ComparisonOperator extends Doctrine_Query_Produc
                 }
             break;
             default:
-                $this->_parser->syntaxError();
+                $this->_parser->logError();
         }
     }
 }

draft/Doctrine/Query/Production/ExistsExpression.php

+<?php
+/*
+ *  $Id$
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This software consists of voluntary contributions made by many individuals
+ * and is licensed under the LGPL. For more information, see
+ * <http://www.phpdoctrine.org>.
+ */
+
+/**
+ * ExistsExpression = ["NOT"] "EXISTS" "(" Subselect ")"
+ *
+ * @package     Doctrine
+ * @subpackage  Query
+ * @author      Janne Vanhala <jpvanhal@cc.hut.fi>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        http://www.phpdoctrine.org
+ * @since       1.0
+ * @version     $Revision$
+ */
+class Doctrine_Query_Production_ExistsExpression extends Doctrine_Query_Production
+{
+    public function execute(array $params = array())
+    {
+        if ($this->_isNextToken(Doctrine_Query_Token::T_NOT)) {
+            $this->_parser->match(Doctrine_Query_Token::T_NOT);
+        }
+
+        $this->_parser->match(Doctrine_Query_Token::T_EXISTS);
+
+        $this->_parser->match('(');
+        $this->Subselect();
+        $this->_parser->match(')');
+    }
+}

draft/Doctrine/Query/Production/Function.php

+<?php
+/*
+ *  $Id$
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This software consists of voluntary contributions made by many individuals
+ * and is licensed under the LGPL. For more information, see
+ * <http://www.phpdoctrine.org>.
+ */
+
+/**
+ * Function = identifier "(" [Expression {"," Expression}] ")"
+ *
+ * @package     Doctrine
+ * @subpackage  Query
+ * @author      Janne Vanhala <jpvanhal@cc.hut.fi>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        http://www.phpdoctrine.org
+ * @since       1.0
+ * @version     $Revision$
+ */
+class Doctrine_Query_Production_Function extends Doctrine_Query_Production
+{
+    public function execute(array $params = array())
+    {
+        $this->_parser->match(Doctrine_Query_Token::T_IDENTIFIER);
+
+        $this->_parser->match('(');
+
+        if ( ! $this->_isNextToken(')')) {
+            $this->Expression();
+            while ($this->_isNextToken(',')) {
+                $this->_parser->match(',');
+                $this->Expression();
+            }
+        }
+
+        $this->_parser->match(')');
+    }
+}

draft/Doctrine/Query/Production/InExpression.php

+<?php
+/*
+ *  $Id$
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This software consists of voluntary contributions made by many individuals
+ * and is licensed under the LGPL. For more information, see
+ * <http://www.phpdoctrine.org>.
+ */
+
+/**
+ * InExpression = ["NOT"] "IN" "(" (Atom {"," Atom} | Subselect) ")"
+ *
+ * @package     Doctrine
+ * @subpackage  Query
+ * @author      Janne Vanhala <jpvanhal@cc.hut.fi>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        http://www.phpdoctrine.org
+ * @since       1.0
+ * @version     $Revision$
+ */
+class Doctrine_Query_Production_InExpression extends Doctrine_Query_Production
+{
+    public function execute(array $params = array())
+    {
+        if ($this->_isNextToken(Doctrine_Query_Token::T_NOT)) {
+            $this->_parser->match(Doctrine_Query_Token::T_NOT);
+        }
+
+        $this->_parser->match(Doctrine_Query_Token::T_IN);
+
+        $this->_parser->match('(');
+
+        if ($this->_isNextToken(Doctrine_Query_Token::T_SELECT)) {
+            $this->Subselect();
+        } else {
+            $this->Atom();
+            while ($this->_isNextToken(',')) {
+                $this->_parser->match(',');
+                $this->Atom();
+            }
+        }
+
+        $this->_parser->match(')');
+    }
+}

draft/Doctrine/Query/Production/Primary.php

@@ -37,36 +37,24 @@ class Doctrine_Query_Production_Primary extends Doctrine_Query_Production
     {
         switch ($this->_parser->lookahead['type']) {
             case Doctrine_Query_Token::T_IDENTIFIER:
-                // @todo: custom functions
+                $nextToken = $this->_parser->getScanner()->peek();
+
+                if ($nextToken['value'] === '(') {
+                    $this->Function();
+                } else {
                     $this->PathExpression();
+                }
             break;
             case Doctrine_Query_Token::T_STRING:
             case Doctrine_Query_Token::T_NUMERIC:
             case Doctrine_Query_Token::T_INPUT_PARAMETER:
                 $this->Atom();
             break;
-            case Doctrine_Query_Token::T_LENGTH:
-            case Doctrine_Query_Token::T_LOCATE:
-            case Doctrine_Query_Token::T_ABS:
-            case Doctrine_Query_Token::T_SQRT:
-            case Doctrine_Query_Token::T_MOD:
-            case Doctrine_Query_Token::T_SIZE:
-            case Doctrine_Query_Token::T_CURRENT_DATE:
-            case Doctrine_Query_Token::T_CURRENT_TIMESTAMP:
-            case Doctrine_Query_Token::T_CURRENT_TIME:
-            case Doctrine_Query_Token::T_SUBSTRING:
-            case Doctrine_Query_Token::T_CONCAT:
-            case Doctrine_Query_Token::T_TRIM:
-            case Doctrine_Query_Token::T_LOWER:
-            case Doctrine_Query_Token::T_UPPER:
-                $this->Function();
-            break;
             case Doctrine_Query_Token::T_AVG:
+            case Doctrine_Query_Token::T_COUNT:
             case Doctrine_Query_Token::T_MAX:
             case Doctrine_Query_Token::T_MIN:
             case Doctrine_Query_Token::T_SUM:
-            case Doctrine_Query_Token::T_MOD:
-            case Doctrine_Query_Token::T_SIZE:
                 $this->AggregateExpression();
             break;
             case Doctrine_Query_Token::T_NONE:
@@ -77,7 +65,7 @@ class Doctrine_Query_Production_Primary extends Doctrine_Query_Production
                     break;
                 }
             default:
-                $this->_parser->syntaxError();
+                $this->_parser->logError();
         }
     }
 }

draft/Doctrine/Query/Production/QuantifiedExpression.php

@@ -45,7 +45,7 @@ class Doctrine_Query_Production_QuantifiedExpression extends Doctrine_Query_Prod
                 $this->_parser->match(Doctrine_Query_Token::T_SOME);
             break;
             default:
-                $this->syntaxError();
+                $this->_parser->logError();
         }
 
         $this->_parser->match('(');

draft/Doctrine/Query/Production/SimpleSelectClause.php

+<?php
+/*
+ *  $Id$
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This software consists of voluntary contributions made by many individuals
+ * and is licensed under the LGPL. For more information, see
+ * <http://www.phpdoctrine.org>.
+ */
+
+/**
+ * SimpleSelectClause = "SELECT" ["DISTINCT"] SelectExpression
+ *
+ * @package     Doctrine
+ * @subpackage  Query
+ * @author      Janne Vanhala <jpvanhal@cc.hut.fi>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        http://www.phpdoctrine.org
+ * @since       1.0
+ * @version     $Revision$
+ */
+class Doctrine_Query_Production_SimpleSelectClause extends Doctrine_Query_Production
+{
+    public function execute(array $params = array())
+    {
+        $this->_parser->match(Doctrine_Query_Token::T_SELECT);
+
+        if ($this->_isNextToken(Doctrine_Query_Token::T_DISTINCT)) {
+            $this->_parser->match(Doctrine_Query_Token::T_DISTINCT);
+        }
+
+        $this->SelectExpression();
+    }
+}

draft/Doctrine/Query/Production/Subselect.php

+<?php
+/*
+ *  $Id$
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This software consists of voluntary contributions made by many individuals
+ * and is licensed under the LGPL. For more information, see
+ * <http://www.phpdoctrine.org>.
+ */
+
+/**
+ * Subselect = SimpleSelectClause FromClause [WhereClause] [GroupByClause]
+ *     [HavingClause] [OrderByClause] [LimitClause] [OffsetClause]
+ *
+ * @package     Doctrine
+ * @subpackage  Query
+ * @author      Janne Vanhala <jpvanhal@cc.hut.fi>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        http://www.phpdoctrine.org
+ * @since       1.0
+ * @version     $Revision$
+ */
+class Doctrine_Query_Production_Subselect extends Doctrine_Query_Production
+{
+    public function execute(array $params = array())
+    {
+        $this->SimpleSelectClause();
+
+        $this->FromClause();
+
+        if ($this->_isNextToken(Doctrine_Query_Token::T_WHERE)) {
+            $this->WhereClause();
+        }
+
+        if ($this->_isNextToken(Doctrine_Query_Token::T_GROUP)) {
+            $this->GroupByClause();
+        }
+
+        if ($this->_isNextToken(Doctrine_Query_Token::T_HAVING)) {
+            $this->HavingClause();
+        }
+
+        if ($this->_isNextToken(Doctrine_Query_Token::T_ORDER)) {
+            $this->OrderByClause();
+        }
+
+        if ($this->_isNextToken(Doctrine_Query_Token::T_LIMIT)) {
+            $this->LimitClause();
+        }
+    }
+}

draft/Doctrine/Query/Scanner.php

@@ -122,7 +122,11 @@ class Doctrine_Query_Scanner
 
     public function peek()
     {
+        if (isset($this->_tokens[$this->_position + $this->_peek])) {
             return $this->_tokens[$this->_position + $this->_peek++];
+        } else {
+            return null;
+        }
     }
 
     public function resetPeek()

draft/Doctrine/Query/Token.php

@@ -75,10 +75,6 @@ final class Doctrine_Query_Token
     const T_SUM                 = 135;
     const T_UPDATE              = 136;
     const T_WHERE               = 137;
-    const T_LENGTH              = 138;
-    const T_LOCATE              = 139;
-    const T_ABS                 = 140;
-    const T_SQRT                = 141;
     const T_MOD                 = 142;
     const T_SIZE                = 143;
 

draft/tests/Query/LanguageRecognitionTestCase.php

@@ -49,4 +49,54 @@ class Doctrine_Query_LanguageRecognition_TestCase extends Doctrine_UnitTestCase
     {
         $this->assertValidDql('SELECT u.*, p.* FROM User u, u.Phonenumber p');
     }
+
+    public function testSelectDistinctIsSupported()
+    {
+        $this->assertValidDql('SELECT DISTINCT u.name FROM User u');
+    }
+
+    public function testAggregateFunctionInSelect()
+    {
+        $this->assertValidDql('SELECT COUNT(u.id) FROM User u');
+    }
+
+    public function testAggregateFunctionWithDistinctInSelect()
+    {
+        $this->assertValidDql('SELECT COUNT(DISTINCT u.name) FROM User u');
+    }
+
+    public function testFunctionalExpressionsSupportedInWherePart()
+    {
+        $this->assertValidDql("SELECT u.name FROM User u WHERE TRIM(u.name) = 'someone'");
+    }
+
+    public function testArithmeticExpressionsSupportedInWherePart()
+    {
+        $this->assertValidDql('FROM Account a WHERE ((a.amount + 5000) * a.amount + 3) < 10000000');
+    }
+
+    public function testInExpressionSupportedInWherePart()
+    {
+        $this->assertValidDql('FROM User WHERE User.id IN (1, 2)');
+    }
+
+    public function testNotInExpressionSupportedInWherePart()
+    {
+        $this->assertValidDql('FROM User WHERE User.id NOT IN (1)');
+    }
+
+    public function testExistsExpressionSupportedInWherePart()
+    {
+        $this->assertValidDql('FROM User WHERE EXISTS (SELECT g.id FROM Groupuser g WHERE g.user_id = u.id)');
+    }
+
+    public function testNotExistsExpressionSupportedInWherePart()
+    {
+        $this->assertValidDql('FROM User WHERE NOT EXISTS (SELECT g.id FROM Groupuser g WHERE g.user_id = u.id)');
+    }
+
+    public function testLiteralValueAsInOperatorOperandIsSupported()
+    {
+        $this->assertValidDql('SELECT u.id FROM User u WHERE 1 IN (1, 2)');
+    }
 }