Skip to content

D
doctrine-dbal
Commit 88698c3a authored by guilhermeblanco's avatar guilhermeblanco
Browse files
Options

[2.0] Created common Lexer code for Annotations parser and Query parser. Added some docblocks

parent 084add0a
Show whitespace changes
Inline Side-by-side
Showing with 419 additions and 424 deletions
lib/Doctrine/Common/Annotations/Annotation.php
View file @ 88698c3a
<?php
/*
* $Id$
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This software consists of voluntary contributions made by many individuals
* and is licensed under the LGPL. For more information, see
* <http://www.doctrine-project.org>.
*/
namespace Doctrine\Common\Annotations;
/**
* Annotations class
*
* @license http://www.opensource.org/licenses/lgpl-license.php LGPL
* @link www.doctrine-project.org
* @since 2.0
* @version $Revision: 3938 $
* @author Guilherme Blanco <guilhermeblanco@hotmail.com>
* @author Jonathan Wage <jonwage@gmail.com>
* @author Roman Borschel <roman@code-factory.org>
*/
class Annotation
{
/**
* Value property. Common among all derived classes.
*
* @var string
*/
public $value;
/**
* Constructor
*
* @param array $data Key-value for properties to be defined in this class
*/
public final function __construct(array $data)
{
foreach ($data as $key => $value) {
......
lib/Doctrine/Common/Annotations/AnnotationReader.php
View file @ 88698c3a
......@@ -21,23 +21,49 @@
namespace Doctrine\Common\Annotations;
use \ReflectionClass, \ReflectionMethod, \ReflectionProperty;
use Doctrine\Common\Cache\Cache;
use \ReflectionClass,
\ReflectionMethod,
\ReflectionProperty,
Doctrine\Common\Cache\Cache;
/**
* A reader for docblock annotations.
*
* @author Roman Borschel <roman@code-factory.org>
* @license http://www.opensource.org/licenses/lgpl-license.php LGPL
* @link www.doctrine-project.org
* @since 2.0
* @version $Revision: 3938 $
* @author Guilherme Blanco <guilhermeblanco@hotmail.com>
* @author Jonathan Wage <jonwage@gmail.com>
* @author Roman Borschel <roman@code-factory.org>
*/
class AnnotationReader
{
/**
* Cache salt
*
* @var string
* @static
*/
private static $CACHE_SALT = "@<Annot>";
/**
* Annotations Parser
*
* @var Doctrine\Common\Annotations\Parser
*/
private $_parser;
/**
* Cache machanism to store processed Annotations
*
* @var Doctrine\Common\Cache\Cache
*/
private $_cache;
/**
* Initiaizes a new AnnotationReader that uses the given Cache provider to cache annotations.
* Constructor. Initiaizes a new AnnotationReader that uses the given
* Cache provider to cache annotations.
*
* @param Cache $cache The cache provider to use.
*/
......
lib/Doctrine/Common/Annotations/Lexer.php
View file @ 88698c3a
......@@ -24,13 +24,15 @@ namespace Doctrine\Common\Annotations;
/**
* Simple lexer for docblock annotations.
*
* @author Roman Borschel <roman@code-factory.org>
* @license http://www.opensource.org/licenses/lgpl-license.php LGPL
* @link www.doctrine-project.org
* @since 2.0
* @version $Revision$
* @version $Revision: 3938 $
* @author Guilherme Blanco <guilhermeblanco@hotmail.com>
* @author Jonathan Wage <jonwage@gmail.com>
* @author Roman Borschel <roman@code-factory.org>
*/
class Lexer
class Lexer extends \Doctrine\Common\Lexer
{
const T_NONE = 1;
const T_FLOAT = 2;
......@@ -41,138 +43,30 @@ class Lexer
const T_FALSE = 7;
/**
* Array of scanned tokens.
*
* @var array
*/
private $_tokens = array();
private $_position = 0;
private $_peek = 0;
/**
* @var array The next token in the query string.
*/
public $lookahead;
/**
* @var array The last matched/seen token.
*/
public $token;
public function setInput($input)
{
$this->_tokens = array();
$this->_scan($input);
}
public function reset()
{
$this->lookahead = null;
$this->token = null;
$this->_peek = 0;
$this->_position = 0;
}
/**
* Checks whether a given token matches the current lookahead.
*
* @param integer|string $token
* @return boolean
*/
public function isNextToken($token)
{
$la = $this->lookahead;
return ($la['type'] === $token || $la['value'] === $token);
}
/**
* Moves to the next token in the input string.
*
* A token is an associative array containing three items:
* - 'value' : the string value of the token in the input string
* - 'type' : the type of the token (identifier, numeric, string, input
* parameter, none)
* - 'position' : the position of the token in the input string
*
* @return array|null the next token; null if there is no more tokens left
*/
public function moveNext()
{
$this->token = $this->lookahead;
$this->_peek = 0;
if (isset($this->_tokens[$this->_position])) {
$this->lookahead = $this->_tokens[$this->_position++];
return true;
} else {
$this->lookahead = null;
return false;
}
}
/**
* Tells the lexer to skip input tokens until it sees a token with the given value.
*
* @param $value The value to skip until.
*/
public function skipUntil($value)
{
while ($this->lookahead !== null && $this->lookahead['value'] !== $value) {
$this->moveNext();
}
}
/**
* Checks if an identifier is a keyword and returns its correct type.
*
* @param string $identifier identifier name
* @return int token type
* @inheritdoc
*/
private function _checkLiteral($identifier)
protected function getCatchablePatterns()
{
$name = 'Doctrine\Common\Annotations\Lexer::T_' . strtoupper($identifier);
if (defined($name)) {
return constant($name);
}
return self::T_IDENTIFIER;
}
/**
* Scans the input string for tokens.
*
* @param string $input a query string
*/
private function _scan($input)
{
static $regex;
if ( ! isset($regex)) {
$patterns = array(
return array(
'[a-z_][a-z0-9_\\\]*',
'(?:[0-9]+(?:[\.][0-9]+)*)(?:e[+-]?[0-9]+)?',
'"(?:[^"]|"")*"'
);
$regex = '/(' . implode(')|(', $patterns) . ')|\s+|\*+|(.)/i';
}
$matches = preg_split($regex, $input, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
foreach ($matches as $match) {
$type = $this->_getType($match);
$this->_tokens[] = array(
'value' => $match,
'type' => $type
);
}
/**
* @inheritdoc
*/
protected function getNonCatchablePatterns()
{
return array('\s+', '\*+', '(.)');
}
/**
* @todo Doc
* @inheritdoc
*/
private function _getType(&$value)
protected function _getType(&$value)
{
// $value is referenced because it can be changed if it is numeric.
$type = self::T_NONE;
$newVal = $this->_getNumeric($value);
......@@ -211,28 +105,19 @@ class Lexer
}
/**
* Moves the lookahead token forward.
* Checks if an identifier is a keyword and returns its correct type.
*
* @return array|null The next token or NULL if there are no more tokens ahead.
* @param string $identifier identifier name
* @return int token type
*/
public function peek()
private function _checkLiteral($identifier)
{
if (isset($this->_tokens[$this->_position + $this->_peek])) {
return $this->_tokens[$this->_position + $this->_peek++];
} else {
return null;
}
$name = 'Doctrine\Common\Annotations\Lexer::T_' . strtoupper($identifier);
if (defined($name)) {
return constant($name);
}
/**
* Peeks at the next token, returns it and immediately resets the peek.
*
* @return array|null The next token or NULL if there are no more tokens ahead.
*/
public function glimpse()
{
$peek = $this->peek();
$this->_peek = 0;
return $peek;
return self::T_IDENTIFIER;
}
}
\ No newline at end of file
lib/Doctrine/Common/Lexer.php 0 → 100644
View file @ 88698c3a
<?php
/*
* $Id$
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This software consists of voluntary contributions made by many individuals
* and is licensed under the LGPL. For more information, see
* <http://www.doctrine-project.org>.
*/
namespace Doctrine\Common;
/**
* Simple generic lexical scanner
*
* @license http://www.opensource.org/licenses/lgpl-license.php LGPL
* @link www.doctrine-project.org
* @since 2.0
* @version $Revision: 3938 $
* @author Guilherme Blanco <guilhermeblanco@hotmail.com>
* @author Jonathan Wage <jonwage@gmail.com>
* @author Roman Borschel <roman@code-factory.org>
*/
abstract class Lexer
{
/**
* Array of scanned tokens.
*
* @var array
*/
private $_tokens = array();
/**
* @todo Doc
*/
private $_position = 0;
/**
* @todo Doc
*/
private $_peek = 0;
/**
* @var array The next token in the query string.
*/
public $lookahead;
/**
* @var array The last matched/seen token.
*/
public $token;
/**
* Inputs data to be tokenized
*
* @param string $input input to be tokenized
*/
public function setInput($input)
{
$this->_tokens = array();
$this->reset();
$this->_scan($input);
}
/**
* Resets the scanner
*
*/
public function reset()
{
$this->lookahead = null;
$this->token = null;
$this->_peek = 0;
$this->_position = 0;
}
/**
* Resets the peek pointer to 0
*
*/
public function resetPeek()
{
$this->_peek = 0;
}
/**
* Resets the lexer position on the input to the given position
*
* @param integer $position Position to place the lexical scanner
*/
public function resetPosition($position = 0)
{
$this->_position = $position;
}
/**
* Checks whether a given token matches the current lookahead.
*
* @param integer|string $token
* @return boolean
*/
public function isNextToken($token)
{
$la = $this->lookahead;
return ($la['type'] === $token || $la['value'] === $token);
}
/**
* Moves to the next token in the input string.
*
* A token is an associative array containing three items:
* - 'value' : the string value of the token in the input string
* - 'type' : the type of the token (identifier, numeric, string, input
* parameter, none)
* - 'position' : the position of the token in the input string
*
* @return array|null the next token; null if there is no more tokens left
*/
public function moveNext()
{
$this->token = $this->lookahead;
$this->_peek = 0;
if (isset($this->_tokens[$this->_position])) {
$this->lookahead = $this->_tokens[$this->_position++];
return true;
} else {
$this->lookahead = null;
return false;
}
}
/**
* Tells the lexer to skip input tokens until it sees a token with the given value.
*
* @param $value The value to skip until.
*/
public function skipUntil($value)
{
while ($this->lookahead !== null && $this->lookahead['value'] !== $value) {
$this->moveNext();
}
}
/**
* @todo Doc
*/
public function isA($value, $token)
{
$type = $this->_getType($value);
return $type === $token;
}
/**
* Moves the lookahead token forward.
*
* @return array | null The next token or NULL if there are no more tokens ahead.
*/
public function peek()
{
if (isset($this->_tokens[$this->_position + $this->_peek])) {
return $this->_tokens[$this->_position + $this->_peek++];
} else {
return null;
}
}
/**
* Peeks at the next token, returns it and immediately resets the peek.
*
* @return array|null The next token or NULL if there are no more tokens ahead.
*/
public function glimpse()
{
$peek = $this->peek();
$this->_peek = 0;
return $peek;
}
/**
* Scans the input string for tokens.
*
* @param string $input a query string
*/
protected function _scan($input)
{
static $regex;
if ( ! isset($regex)) {
$regex = '/(' . implode(')|(', $this->getCatchablePatterns()) . ')|'
. implode('|', $this->getNonCatchablePatterns()) . '/i';
}
$flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
$matches = preg_split($regex, $input, -1, $flags);
foreach ($matches as $match) {
$value = $match[0];
$type = $this->_getType($value);
$this->_tokens[] = array(
'value' => $value,
'type' => $type,
'position' => $match[1]
);
}
}
/**
* Lexical catchable patterns
*
* @return array
*/
abstract protected function getCatchablePatterns();
/**
* Lexical non-catchable patterns
*
* @return array
*/
abstract protected function getNonCatchablePatterns();
/**
* Retrieve token type. Also processes the token value if necessary.
*
* @param string $value
* @return integer
*/
abstract protected function _getType(&$value);
}
\ No newline at end of file
lib/Doctrine/ORM/Query/Lexer.php
View file @ 88698c3a
......@@ -32,7 +32,7 @@ namespace Doctrine\ORM\Query;
* @since 2.0
* @version $Revision$
*/
class Lexer
class Lexer extends \Doctrine\Common\Lexer
{
const T_NONE = 1;
const T_IDENTIFIER = 2;
......@@ -94,33 +94,6 @@ class Lexer
private $_keywordsTable;
/**
* Array of scanned tokens.
*
* @var array
*/
private $_tokens = array();
/**
* @todo Doc
*/
private $_position = 0;
/**
* @todo Doc
*/
private $_peek = 0;
/**
* @var array The next token in the query string.
*/
public $lookahead;
/**
* @var array The last matched/seen token.
*/
public $token;
/**
* Creates a new query scanner object.
*
......@@ -128,105 +101,35 @@ class Lexer
*/
public function __construct($input)
{
$this->_scan($input);
}
/**
* Checks whether a given token matches the current lookahead.
*
* @param integer|string $token
* @return boolean
*/
public function isNextToken($token)
{
$la = $this->lookahead;
return ($la['type'] === $token || $la['value'] === $token);
}
/**
* Moves to the next token in the input string.
*
* A token is an associative array containing three items:
* - 'value' : the string value of the token in the input string
* - 'type' : the type of the token (identifier, numeric, string, input
* parameter, none)
* - 'position' : the position of the token in the input string
*
* @return array|null the next token; null if there is no more tokens left
*/
public function moveNext()
{
$this->token = $this->lookahead;
$this->_peek = 0;
if (isset($this->_tokens[$this->_position])) {
$this->lookahead = $this->_tokens[$this->_position++];
return true;
} else {
$this->lookahead = null;
return false;
}
}
/**
* Checks if an identifier is a keyword and returns its correct type.
*
* @param string $identifier identifier name
* @return int token type
*/
private function _checkLiteral($identifier)
{
$name = 'Doctrine\ORM\Query\Lexer::T_' . strtoupper($identifier);
if (defined($name)) {
$type = constant($name);
if ($type > 100) {
return $type;
}
}
return self::T_IDENTIFIER;
$this->setInput($input);
}
/**
* Scans the input string for tokens.
*
* @param string $input a query string
* @inheritdoc
*/
private function _scan($input)
protected function getCatchablePatterns()
{
static $regex;
if ( ! isset($regex)) {
$patterns = array(
return array(
'[a-z_][a-z0-9_\\\]*',
'(?:[0-9]+(?:[,\.][0-9]+)*)(?:e[+-]?[0-9]+)?',
"'(?:[^']|'')*'",
'\?[1-9]+|:[a-z][a-z0-9_]+'
);
$regex = '/(' . implode(')|(', $patterns) . ')|\s+|(.)/i';
}
$flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
$matches = preg_split($regex, $input, -1, $flags);
foreach ($matches as $match) {
$value = $match[0];
$type = $this->_getType($value);
$this->_tokens[] = array(
'value' => $value,
'type' => $type,
'position' => $match[1]
);
}
/**
* @inheritdoc
*/
protected function getNonCatchablePatterns()
{
return array('\s+', '(.)');
}
/**
* @todo Doc
* @inheritdoc
*/
private function _getType(&$value)
protected function _getType(&$value)
{
// $value is referenced because it can be changed if it is numeric.
// [TODO] Revisit the _isNumeric and _getNumeric methods to reduce overhead.
$type = self::T_NONE;
$newVal = $this->_getNumeric($value);
......@@ -263,72 +166,27 @@ class Lexer
return $value;
}
// World number: 1.000.000,02 or -1,234e-2
$worldnum = strtr($value, array('.' => '', ',' => '.'));
if (is_numeric($worldnum)) {
return $worldnum;
}
// American extensive number: 1,000,000.02
$american_en = strtr($value, array(',' => ''));
if (is_numeric($american_en)) {
return $american_en;
}
return false;
}
/**
* @todo Doc
*/
public function isA($value, $token)
{
$type = $this->_getType($value);
return $type === $token;
}
/**
* Moves the lookahead token forward.
* Checks if an identifier is a keyword and returns its correct type.
*
* @return array|null The next token or NULL if there are no more tokens ahead.
* @param string $identifier identifier name
* @return int token type
*/
public function peek()
private function _checkLiteral($identifier)
{
if (isset($this->_tokens[$this->_position + $this->_peek])) {
return $this->_tokens[$this->_position + $this->_peek++];
} else {
return null;
}
}
$name = 'Doctrine\ORM\Query\Lexer::T_' . strtoupper($identifier);
/**
* Peeks at the next token, returns it and immediately resets the peek.
*
* @return array|null The next token or NULL if there are no more tokens ahead.
*/
public function glimpse()
{
$peek = $this->peek();
$this->_peek = 0;
return $peek;
if (defined($name)) {
$type = constant($name);
if ($type > 100) {
return $type;
}
/**
* Resets the peek pointer to 0.
*/
public function resetPeek()
{
$this->_peek = 0;
}
/**
* Resets the lexer position on the input to the given position.
*/
public function resetPosition($position = 0)
{
$this->_position = $position;
return self::T_IDENTIFIER;
}
/**
......
tests/Doctrine/Tests/ORM/Query/LexerTest.php
View file @ 88698c3a
......@@ -100,65 +100,11 @@ class LexerTest extends \Doctrine\Tests\OrmTestCase
public function testScannerRecognizesFloatBig()
{
$lexer = new Lexer('1,234,567.89');
$lexer = new Lexer('123456789.01');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.23456789e6, $token['value']);
}
public function testScannerRecognizesFloatBigWrongPoint()
{
$lexer = new Lexer('12,34,56,7.89');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.23456789e6, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecific()
{
$lexer = new Lexer('1,234');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.234, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecificBig()
{
$lexer = new Lexer('1.234.567,89');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.23456789e6, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecificBigWrongPoint()
{
$lexer = new Lexer('12.34.56.7,89');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.23456789e6, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecificExponent()
{
$lexer = new Lexer('1,234e2');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(1.234e2, $token['value']);
}
public function testScannerRecognizesFloatLocaleSpecificExponent2()
{
$lexer = new Lexer('0,234e2');
$lexer->moveNext();
$token = $lexer->lookahead;
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
$this->assertEquals(.234e2, $token['value']);
$this->assertEquals(1.2345678901e8, $token['value']);
}
public function testScannerRecognizesFloatContainingWhitespace()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment