[2.0] Refactored and reenabled Lexer tests.

lib/Doctrine/ORM/Query/Lexer.php

@@ -114,7 +114,7 @@ class Lexer
     public $lookahead;
 
     /**
-     * @var array The last matched token.
+     * @var array The last matched/seen token.
      */
     public $token;
 
@@ -151,17 +151,43 @@ class Lexer
      *
      * @return array|null the next token; null if there is no more tokens left
      */
-    public function next()
+    public function moveNext()
     {
         $this->token = $this->lookahead;
         $this->_peek = 0;
         if (isset($this->_tokens[$this->_position])) {
             $this->lookahead = $this->_tokens[$this->_position++];
+            return true;
         } else {
             $this->lookahead = null;
+            return false;
+        }
+    }
+
+    /**
+     * Attempts to match the given token with the current lookahead token.
+     *
+     * If they match, the lexer moves on to the next token, otherwise a syntax error
+     * is raised.
+     *
+     * @param int|string token type or value
+     * @return bool True, if tokens match; false otherwise.
+     */
+    /*public function match($token)
+    {
+        if (is_string($token)) {
+            $isMatch = ($this->lookahead['value'] === $token);
+        } else {
+            $isMatch = ($this->lookahead['type'] === $token);
         }
+
+        if ( ! $isMatch) {
+            $this->syntaxError($this->getLiteral($token));
         }
 
+        $this->moveNext();
+    }*/
+
     /**
      * Checks if an identifier is a keyword and returns its correct type.
      *

lib/Doctrine/ORM/Query/Parser.php

@@ -60,21 +60,21 @@ class Parser
      *
      * @var Doctrine_ORM_Query_Scanner
      */
-    protected $_lexer;
+    private $_lexer;
 
     /**
      * The Parser Result object.
      *
      * @var Doctrine_ORM_Query_ParserResult
      */
-    protected $_parserResult;
+    private $_parserResult;
     
     /**
      * The EntityManager.
      *
      * @var EnityManager
      */
-    protected $_em;
+    private $_em;
 
     /**
      * Creates a new query parser object.
@@ -131,8 +131,7 @@ class Parser
             $this->syntaxError($this->_lexer->getLiteral($token));
         }
 
-        $this->_lexer->next();
-        return true;
+        $this->_lexer->moveNext();
     }
 
     public function isA($value, $token)
@@ -322,7 +321,7 @@ class Parser
      */
     private function _QueryLanguage()
     {
-        $this->_lexer->next();
+        $this->_lexer->moveNext();
         switch ($this->_lexer->lookahead['type']) {
             case Lexer::T_SELECT:
                 return $this->_SelectStatement();

tests/Doctrine/Tests/ORM/Query/AllTests.php

@@ -22,9 +22,10 @@ class AllTests
         $suite->addTestSuite('Doctrine\Tests\ORM\Query\IdentifierRecognitionTest');
         $suite->addTestSuite('Doctrine\Tests\ORM\Query\SelectSqlGenerationTest');
         $suite->addTestSuite('Doctrine\Tests\ORM\Query\LanguageRecognitionTest');
+        $suite->addTestSuite('Doctrine\Tests\ORM\Query\LexerTest');
         
         /*
-        $suite->addTestSuite('Orm_Query_ScannerTest');
+        
         $suite->addTestSuite('Orm_Query_DqlGenerationTest');
         $suite->addTestSuite('Orm_Query_DeleteSqlGenerationTest');
         $suite->addTestSuite('Orm_Query_UpdateSqlGenerationTest');*/

tests/Doctrine/Tests/ORM/Query/ScannerTest.php → tests/Doctrine/Tests/ORM/Query/LexerTest.php

 <?php
-class Orm_Query_ScannerTest extends Doctrine_OrmTestCase
+
+namespace Doctrine\Tests\ORM\Query;
+
+use Doctrine\ORM\Query\Lexer;
+
+require_once __DIR__ . '/../../TestInit.php';
+
+class LexerTest extends \Doctrine\Tests\OrmTestCase
 {
+    //private $_lexer;
+
+    protected function setUp() {
+    }
+
     public function testScannerRecognizesIdentifierWithLengthOfOneCharacter()
     {
-        $scanner = new Doctrine_Query_Scanner('u');
+        $lexer = new Lexer('u');
         
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+
+        $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
         $this->assertEquals('u', $token['value']);
     }
 
     public function testScannerRecognizesIdentifierConsistingOfLetters()
     {
-        $scanner = new Doctrine_Query_Scanner('someIdentifier');
+        $lexer = new Lexer('someIdentifier');
 
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
         $this->assertEquals('someIdentifier', $token['value']);
     }
 
     public function testScannerRecognizesIdentifierIncludingDigits()
     {
-        $scanner = new Doctrine_Query_Scanner('s0m31d3nt1f13r');
+        $lexer = new Lexer('s0m31d3nt1f13r');
 
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
         $this->assertEquals('s0m31d3nt1f13r', $token['value']);
     }
 
     public function testScannerRecognizesIdentifierIncludingUnderscore()
     {
-        $scanner = new Doctrine_Query_Scanner('some_identifier');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
+        $lexer = new Lexer('some_identifier');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
         $this->assertEquals('some_identifier', $token['value']);
     }
 
     public function testScannerRecognizesDecimalInteger()
     {
-        $scanner = new Doctrine_Query_Scanner('1234');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_INTEGER, $token['type']);
+        $lexer = new Lexer('1234');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_INTEGER, $token['type']);
         $this->assertEquals(1234, $token['value']);
     }
 
     public function testScannerRecognizesFloat()
     {
-        $scanner = new Doctrine_Query_Scanner('1.234');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('1.234');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(1.234, $token['value']);
     }
 
     public function testScannerRecognizesFloatWithExponent()
     {
-        $scanner = new Doctrine_Query_Scanner('1.2e3');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('1.2e3');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(1.2e3, $token['value']);
     }
 
     public function testScannerRecognizesFloatWithExponent2()
     {
-        $scanner = new Doctrine_Query_Scanner('0.2e3');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('0.2e3');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(.2e3, $token['value']);
     }
 
     public function testScannerRecognizesFloatWithNegativeExponent()
     {
-        $scanner = new Doctrine_Query_Scanner('7E-10');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('7E-10');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(7E-10, $token['value']);
     }
 
     public function testScannerRecognizesFloatBig()
     {
-        $scanner = new Doctrine_Query_Scanner('1,234,567.89');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('1,234,567.89');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(1.23456789e6, $token['value']);
     }
 
     public function testScannerRecognizesFloatBigWrongPoint()
     {
-        $scanner = new Doctrine_Query_Scanner('12,34,56,7.89');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('12,34,56,7.89');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(1.23456789e6, $token['value']);
     }
 
     public function testScannerRecognizesFloatLocaleSpecific()
     {
-        $scanner = new Doctrine_Query_Scanner('1,234');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('1,234');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(1.234, $token['value']);
     }
 
     public function testScannerRecognizesFloatLocaleSpecificBig()
     {
-        $scanner = new Doctrine_Query_Scanner('1.234.567,89');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('1.234.567,89');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(1.23456789e6, $token['value']);
     }
 
     public function testScannerRecognizesFloatLocaleSpecificBigWrongPoint()
     {
-        $scanner = new Doctrine_Query_Scanner('12.34.56.7,89');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('12.34.56.7,89');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(1.23456789e6, $token['value']);
     }
 
     public function testScannerRecognizesFloatLocaleSpecificExponent()
     {
-        $scanner = new Doctrine_Query_Scanner('1,234e2');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('1,234e2');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(1.234e2, $token['value']);
     }
 
     public function testScannerRecognizesFloatLocaleSpecificExponent2()
     {
-        $scanner = new Doctrine_Query_Scanner('0,234e2');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer = new Lexer('0,234e2');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertEquals(.234e2, $token['value']);
     }
 
     public function testScannerRecognizesFloatContainingWhitespace()
     {
-        $scanner = new Doctrine_Query_Scanner('-   1.234e2');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_NONE, $token['type']);
+        $lexer = new Lexer('-   1.234e2');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_NONE, $token['type']);
         $this->assertEquals('-', $token['value']);
 
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
         $this->assertNotEquals(-1.234e2, $token['value']);
         $this->assertEquals(1.234e2, $token['value']);
     }
 
     public function testScannerRecognizesStringContainingWhitespace()
     {
-        $scanner = new Doctrine_Query_Scanner("'This is a string.'");
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_STRING, $token['type']);
+        $lexer = new Lexer("'This is a string.'");
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_STRING, $token['type']);
         $this->assertEquals("'This is a string.'", $token['value']);
     }
 
     public function testScannerRecognizesStringContainingSingleQuotes()
     {
-        $scanner = new Doctrine_Query_Scanner("'abc''defg'''");
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_STRING, $token['type']);
+        $lexer = new Lexer("'abc''defg'''");
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_STRING, $token['type']);
         $this->assertEquals("'abc''defg'''", $token['value']);
     }
 
     public function testScannerRecognizesInputParameter()
     {
-        $scanner = new Doctrine_Query_Scanner('?');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_INPUT_PARAMETER, $token['type']);
-        $this->assertEquals('?', $token['value']);
+        $lexer = new Lexer('?1');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_INPUT_PARAMETER, $token['type']);
+        $this->assertEquals('?1', $token['value']);
     }
 
     public function testScannerRecognizesNamedInputParameter()
     {
-        $scanner = new Doctrine_Query_Scanner(':name');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_INPUT_PARAMETER, $token['type']);
+        $lexer = new Lexer(':name');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_INPUT_PARAMETER, $token['type']);
         $this->assertEquals(':name', $token['value']);
     }
 
     public function testScannerTokenizesASimpleQueryCorrectly()
     {
-        $dql = "SELECT u.* FROM User u WHERE u.name = 'Jack O''Neil'";
-        $scanner = new Doctrine_Query_Scanner($dql);
+        $dql = "SELECT u FROM My\Namespace\User u WHERE u.name = 'Jack O''Neil'";
+        $lexer = new Lexer($dql);
 
         $tokens = array(
             array(
                 'value' => 'SELECT',
-                'type'  => Doctrine_Query_Token::T_SELECT,
+                'type'  => Lexer::T_SELECT,
                 'position' => 0
             ),
             array(
                 'value' => 'u',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
+                'type'  => Lexer::T_IDENTIFIER,
                 'position' => 7
             ),
-            array(
-                'value' => '.',
-                'type'  => Doctrine_Query_Token::T_NONE,
-                'position' => 8
-            ),
-            array(
-                'value' => '*',
-                'type'  => Doctrine_Query_Token::T_NONE,
-                'position' => 9
-            ),
             array(
                 'value' => 'FROM',
-                'type'  => Doctrine_Query_Token::T_FROM,
-                'position' => 11
+                'type'  => Lexer::T_FROM,
+                'position' => 9
             ),
             array(
-                'value' => 'User',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 16
+                'value' => 'My\Namespace\User',
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 14
             ),
             array(
                 'value' => 'u',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 21
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 32
             ),
             array(
                 'value' => 'WHERE',
-                'type'  => Doctrine_Query_Token::T_WHERE,
-                'position' => 23
+                'type'  => Lexer::T_WHERE,
+                'position' => 34
             ),
             array(
                 'value' => 'u',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 29
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 40
             ),
             array(
                 'value' => '.',
-                'type'  => Doctrine_Query_Token::T_NONE,
-                'position' => 30
+                'type'  => Lexer::T_NONE,
+                'position' => 41
             ),
             array(
                 'value' => 'name',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 31
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 42
             ),
             array(
                 'value' => '=',
-                'type'  => Doctrine_Query_Token::T_NONE,
-                'position' => 36
+                'type'  => Lexer::T_NONE,
+                'position' => 47
             ),
             array(
                 'value' => "'Jack O''Neil'",
-                'type'  => Doctrine_Query_Token::T_STRING,
-                'position' => 38
+                'type'  => Lexer::T_STRING,
+                'position' => 49
             )
         );
 
         foreach ($tokens as $expected) {
-            $actual = $scanner->next();
+            $lexer->moveNext();
+            $actual = $lexer->lookahead;
             $this->assertEquals($expected['value'], $actual['value']);
             $this->assertEquals($expected['type'], $actual['type']);
             $this->assertEquals($expected['position'], $actual['position']);
         }
 
-        $this->assertNull($scanner->next());
+        $this->assertFalse($lexer->moveNext());
     }
 }