Skip to content

D
doctrine-dbal
Commit 89a62502 authored by romanb's avatar romanb
Browse files
Options

[2.0] Refactored and reenabled Lexer tests.

parent b718cd1a
Hide whitespace changes
Inline Side-by-side
Showing with 323 additions and 9 deletions
lib/Doctrine/ORM/Query/Lexer.php
View file @ 89a62502
......@@ -114,7 +114,7 @@ class Lexer
public $lookahead;
/**
* @var array The last matched token.
* @var array The last matched/seen token.
*/
public $token;
......@@ -151,17 +151,43 @@ class Lexer
*
* @return array|null the next token; null if there is no more tokens left
*/
public function next()
public function moveNext()
{
$this->token = $this->lookahead;
$this->_peek = 0;
if (isset($this->_tokens[$this->_position])) {
$this->lookahead = $this->_tokens[$this->_position++];
return true;
} else {
$this->lookahead = null;
return false;
}
}
/**
* Attempts to match the given token with the current lookahead token.
*
* If they match, the lexer moves on to the next token, otherwise a syntax error
* is raised.
*
* @param int|string token type or value
* @return bool True, if tokens match; false otherwise.
*/
/*public function match($token)
{
if (is_string($token)) {
$isMatch = ($this->lookahead['value'] === $token);
} else {
$isMatch = ($this->lookahead['type'] === $token);
}
if ( ! $isMatch) {
$this->syntaxError($this->getLiteral($token));
}
$this->moveNext();
}*/
/**
* Checks if an identifier is a keyword and returns its correct type.
*
......
lib/Doctrine/ORM/Query/Parser.php
View file @ 89a62502
......@@ -60,21 +60,21 @@ class Parser
*
* @var Doctrine_ORM_Query_Scanner
*/
protected $_lexer;
private $_lexer;
/**
* The Parser Result object.
*
* @var Doctrine_ORM_Query_ParserResult
*/
protected $_parserResult;
private $_parserResult;
/**
* The EntityManager.
*
* @var EnityManager
*/
protected $_em;
private $_em;
/**
* Creates a new query parser object.
......@@ -131,8 +131,7 @@ class Parser
$this->syntaxError($this->_lexer->getLiteral($token));
}
$this->_lexer->next();
return true;
$this->_lexer->moveNext();
}
public function isA($value, $token)
......@@ -322,7 +321,7 @@ class Parser
*/
private function _QueryLanguage()
{
$this->_lexer->next();
$this->_lexer->moveNext();
switch ($this->_lexer->lookahead['type']) {
case Lexer::T_SELECT:
return $this->_SelectStatement();
......
tests/Doctrine/Tests/ORM/Query/AllTests.php
View file @ 89a62502
......@@ -22,9 +22,10 @@ class AllTests
$suite->addTestSuite('Doctrine\Tests\ORM\Query\IdentifierRecognitionTest');
$suite->addTestSuite('Doctrine\Tests\ORM\Query\SelectSqlGenerationTest');
$suite->addTestSuite('Doctrine\Tests\ORM\Query\LanguageRecognitionTest');
$suite->addTestSuite('Doctrine\Tests\ORM\Query\LexerTest');
/*
$suite->addTestSuite('Orm_Query_ScannerTest');
$suite->addTestSuite('Orm_Query_DqlGenerationTest');
$suite->addTestSuite('Orm_Query_DeleteSqlGenerationTest');
$suite->addTestSuite('Orm_Query_UpdateSqlGenerationTest');*/
......
tests/Doctrine/Tests/ORM/Query/ScannerTest.php → tests/Doctrine/Tests/ORM/Query/LexerTest.php
View file @ 89a62502
<?php
class Orm_Query_ScannerTest extends Doctrine_OrmTestCase
namespace Doctrine\Tests\ORM\Query;
use Doctrine\ORM\Query\Lexer;
require_once __DIR__ . '/../../TestInit.php';
class LexerTest extends \Doctrine\Tests\OrmTestCase
{
//private $_lexer;
protected function setUp() {
}
public function testScannerRecognizesIdentifierWithLengthOfOneCharacter()
{
$scanner = new Doctrine_Query_Scanner('u');
$lexer = new Lexer('u');
$lexer->moveNext();
$token = $lexer->lookahead;
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
$this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
$this->assertEquals('u', $token['value']);
}
public function testScannerRecognizesIdentifierConsistingOfLetters()
{
$scanner = new Doctrine_Query_Scanner('someIdentifier');
$lexer = new Lexer('someIdentifier');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
$this->assertEquals('someIdentifier', $token['value']);
}
public function testScannerRecognizesIdentifierIncludingDigits()
{
$scanner = new Doctrine_Query_Scanner('s0m31d3nt1f13r');
$lexer = new Lexer('s0m31d3nt1f13r');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
$this->assertEquals('s0m31d3nt1f13r', $token['value']);
}
public function testScannerRecognizesIdentifierIncludingUnderscore()
{
$scanner = new Doctrine_Query_Scanner('some_identifier');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
$lexer = new Lexer('some_identifier');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
$this->assertEquals('some_identifier', $token['value']);
}
public function testScannerRecognizesDecimalInteger()
{
$scanner = new Doctrine_Query_Scanner('1234');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_INTEGER, $token['type']);
$lexer = new Lexer('1234');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_INTEGER, $token['type']);
$this->assertEquals(1234, $token['value']);
}
public function testScannerRecognizesFloat()
{
$scanner = new Doctrine_Query_Scanner('1.234');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('1.234');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.234, $token['value']);
}
public function testScannerRecognizesFloatWithExponent()
{
$scanner = new Doctrine_Query_Scanner('1.2e3');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('1.2e3');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.2e3, $token['value']);
}
public function testScannerRecognizesFloatWithExponent2()
{
$scanner = new Doctrine_Query_Scanner('0.2e3');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('0.2e3');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(.2e3, $token['value']);
}
public function testScannerRecognizesFloatWithNegativeExponent()
{
$scanner = new Doctrine_Query_Scanner('7E-10');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('7E-10');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(7E-10, $token['value']);
}
public function testScannerRecognizesFloatBig()
{
$scanner = new Doctrine_Query_Scanner('1,234,567.89');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('1,234,567.89');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.23456789e6, $token['value']);
}
public function testScannerRecognizesFloatBigWrongPoint()
{
$scanner = new Doctrine_Query_Scanner('12,34,56,7.89');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('12,34,56,7.89');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.23456789e6, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecific()
{
$scanner = new Doctrine_Query_Scanner('1,234');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('1,234');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.234, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecificBig()
{
$scanner = new Doctrine_Query_Scanner('1.234.567,89');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('1.234.567,89');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.23456789e6, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecificBigWrongPoint()
{
$scanner = new Doctrine_Query_Scanner('12.34.56.7,89');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('12.34.56.7,89');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.23456789e6, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecificExponent()
{
$scanner = new Doctrine_Query_Scanner('1,234e2');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('1,234e2');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.234e2, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecificExponent2()
{
$scanner = new Doctrine_Query_Scanner('0,234e2');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer = new Lexer('0,234e2');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(.234e2, $token['value']);
}
public function testScannerRecognizesFloatContainingWhitespace()
{
$scanner = new Doctrine_Query_Scanner('- 1.234e2');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_NONE, $token['type']);
$lexer = new Lexer('- 1.234e2');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_NONE, $token['type']);
$this->assertEquals('-', $token['value']);
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertNotEquals(-1.234e2, $token['value']);
$this->assertEquals(1.234e2, $token['value']);
}
public function testScannerRecognizesStringContainingWhitespace()
{
$scanner = new Doctrine_Query_Scanner("'This is a string.'");
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_STRING, $token['type']);
$lexer = new Lexer("'This is a string.'");
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_STRING, $token['type']);
$this->assertEquals("'This is a string.'", $token['value']);
}
public function testScannerRecognizesStringContainingSingleQuotes()
{
$scanner = new Doctrine_Query_Scanner("'abc''defg'''");
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_STRING, $token['type']);
$lexer = new Lexer("'abc''defg'''");
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_STRING, $token['type']);
$this->assertEquals("'abc''defg'''", $token['value']);
}
public function testScannerRecognizesInputParameter()
{
$scanner = new Doctrine_Query_Scanner('?');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_INPUT_PARAMETER, $token['type']);
$this->assertEquals('?', $token['value']);
$lexer = new Lexer('?1');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_INPUT_PARAMETER, $token['type']);
$this->assertEquals('?1', $token['value']);
}
public function testScannerRecognizesNamedInputParameter()
{
$scanner = new Doctrine_Query_Scanner(':name');
$token = $scanner->next();
$this->assertEquals(Doctrine_Query_Token::T_INPUT_PARAMETER, $token['type']);
$lexer = new Lexer(':name');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_INPUT_PARAMETER, $token['type']);
$this->assertEquals(':name', $token['value']);
}
public function testScannerTokenizesASimpleQueryCorrectly()
{
$dql = "SELECT u.* FROM User u WHERE u.name = 'Jack O''Neil'";
$scanner = new Doctrine_Query_Scanner($dql);
$dql = "SELECT u FROM My\Namespace\User u WHERE u.name = 'Jack O''Neil'";
$lexer = new Lexer($dql);
$tokens = array(
array(
'value' => 'SELECT',
'type' => Doctrine_Query_Token::T_SELECT,
'type' => Lexer::T_SELECT,
'position' => 0
),
array(
'value' => 'u',
'type' => Doctrine_Query_Token::T_IDENTIFIER,
'type' => Lexer::T_IDENTIFIER,
'position' => 7
),
array(
'value' => '.',
'type' => Doctrine_Query_Token::T_NONE,
'position' => 8
),
array(
'value' => '*',
'type' => Doctrine_Query_Token::T_NONE,
'position' => 9
),
array(
'value' => 'FROM',
'type' => Doctrine_Query_Token::T_FROM,
'position' => 11
'type' => Lexer::T_FROM,
'position' => 9
),
array(
'value' => 'User',
'type' => Doctrine_Query_Token::T_IDENTIFIER,
'position' => 16
'value' => 'My\Namespace\User',
'type' => Lexer::T_IDENTIFIER,
'position' => 14
),
array(
'value' => 'u',
'type' => Doctrine_Query_Token::T_IDENTIFIER,
'position' => 21
'type' => Lexer::T_IDENTIFIER,
'position' => 32
),
array(
'value' => 'WHERE',
'type' => Doctrine_Query_Token::T_WHERE,
'position' => 23
'type' => Lexer::T_WHERE,
'position' => 34
),
array(
'value' => 'u',
'type' => Doctrine_Query_Token::T_IDENTIFIER,
'position' => 29
'type' => Lexer::T_IDENTIFIER,
'position' => 40
),
array(
'value' => '.',
'type' => Doctrine_Query_Token::T_NONE,
'position' => 30
'type' => Lexer::T_NONE,
'position' => 41
),
array(
'value' => 'name',
'type' => Doctrine_Query_Token::T_IDENTIFIER,
'position' => 31
'type' => Lexer::T_IDENTIFIER,
'position' => 42
),
array(
'value' => '=',
'type' => Doctrine_Query_Token::T_NONE,
'position' => 36
'type' => Lexer::T_NONE,
'position' => 47
),
array(
'value' => "'Jack O''Neil'",
'type' => Doctrine_Query_Token::T_STRING,
'position' => 38
'type' => Lexer::T_STRING,
'position' => 49
)
);
foreach ($tokens as $expected) {
$actual = $scanner->next();
$lexer->moveNext();
$actual = $lexer->lookahead;
$this->assertEquals($expected['value'], $actual['value']);
$this->assertEquals($expected['type'], $actual['type']);
$this->assertEquals($expected['position'], $actual['position']);
}
$this->assertNull($scanner->next());
$this->assertFalse($lexer->moveNext());
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment