Lexer.php 6.85 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
<?php
/*
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * This software consists of voluntary contributions made by many individuals
 * and is licensed under the LGPL. For more information, see
 * <http://www.doctrine-project.org>.
 */
 
namespace Doctrine\Common;

/**
23
 * Base class for writing simple lexers, i.e. for creating small DSLs.
24 25 26 27 28
 *
 * @since   2.0
 * @author  Guilherme Blanco <guilhermeblanco@hotmail.com>
 * @author  Jonathan Wage <jonwage@gmail.com>
 * @author  Roman Borschel <roman@code-factory.org>
29
 * @todo Rename: AbstractLexer
30 31 32 33
 */
abstract class Lexer
{
    /**
34
     * @var array Array of scanned tokens
35 36 37 38
     */
    private $_tokens = array();

    /**
39
     * @var integer Current lexer position in input string
40 41 42 43
     */
    private $_position = 0;

    /**
44
     * @var integer Current peek of current lexer position
45 46 47 48
     */
    private $_peek = 0;

    /**
49
     * @var array The next token in the input.
50 51 52 53 54 55 56 57 58
     */
    public $lookahead;

    /**
     * @var array The last matched/seen token.
     */
    public $token;
    
    /**
59 60 61 62
     * Sets the input data to be tokenized.
     *
     * The Lexer is immediately reset and the new input tokenized.
     * Any unprocessed tokens from any previous input are lost.
63
     *
64
     * @param string $input The input to be tokenized.
65 66 67 68 69 70 71 72 73
     */
    public function setInput($input)
    {
        $this->_tokens = array();
        $this->reset();
        $this->_scan($input);
    }
    
    /**
74
     * Resets the lexer.
75 76 77 78
     */
    public function reset()
    {
        $this->lookahead = null;
79 80
        $this->token = null;
        $this->_peek = 0;
81 82
        $this->_position = 0;
    }
83

84
    /**
85
     * Resets the peek pointer to 0.
86 87 88 89 90 91 92
     */
    public function resetPeek()
    {
        $this->_peek = 0;
    }

    /**
93
     * Resets the lexer position on the input to the given position.
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
     *
     * @param integer $position Position to place the lexical scanner
     */
    public function resetPosition($position = 0)
    {
        $this->_position = $position;
    }
    
    /**
     * Checks whether a given token matches the current lookahead.
     *
     * @param integer|string $token
     * @return boolean
     */
    public function isNextToken($token)
    {
110
        return $this->lookahead['type'] === $token;
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
    }

    /**
     * Moves to the next token in the input string.
     *
     * A token is an associative array containing three items:
     *  - 'value'    : the string value of the token in the input string
     *  - 'type'     : the type of the token (identifier, numeric, string, input
     *                 parameter, none)
     *  - 'position' : the position of the token in the input string
     *
     * @return array|null the next token; null if there is no more tokens left
     */
    public function moveNext()
    {
        $this->_peek = 0;
127 128 129 130 131
        $this->token = $this->lookahead;
        $this->lookahead = (isset($this->_tokens[$this->_position]))
            ? $this->_tokens[$this->_position++] : null;
        
        return $this->lookahead !== null;
132 133 134 135 136
    }
    
    /**
     * Tells the lexer to skip input tokens until it sees a token with the given value.
     * 
137
     * @param $type The token type to skip until.
138
     */
139
    public function skipUntil($type)
140
    {
141
        while ($this->lookahead !== null && $this->lookahead['type'] !== $type) {
142 143 144 145 146
            $this->moveNext();
        }
    }
    
    /**
147 148 149 150 151
     * Checks if given value is identical to the given token
     *
     * @param mixed $value
     * @param integer $token
     * @return boolean
152 153 154
     */
    public function isA($value, $token)
    {
155
        return $this->_getType($value) === $token;
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
    }

    /**
     * Moves the lookahead token forward.
     *
     * @return array | null The next token or NULL if there are no more tokens ahead.
     */
    public function peek()
    {
        if (isset($this->_tokens[$this->_position + $this->_peek])) {
            return $this->_tokens[$this->_position + $this->_peek++];
        } else {
            return null;
        }
    }

    /**
     * Peeks at the next token, returns it and immediately resets the peek.
     *
     * @return array|null The next token or NULL if there are no more tokens ahead.
     */
    public function glimpse()
    {
        $peek = $this->peek();
        $this->_peek = 0;
        return $peek;
    }
    
    /**
     * Scans the input string for tokens.
     *
     * @param string $input a query string
     */
    protected function _scan($input)
    {
        static $regex;

        if ( ! isset($regex)) {
            $regex = '/(' . implode(')|(', $this->getCatchablePatterns()) . ')|' 
                   . implode('|', $this->getNonCatchablePatterns()) . '/i';
        }

        $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
        $matches = preg_split($regex, $input, -1, $flags);

        foreach ($matches as $match) {
202 203 204
            // Must remain before 'value' assignment since it can change content
            $type = $this->_getType($match[0]);
            
205
            $this->_tokens[] = array(
206
                'value' => $match[0],
207 208
                'type'  => $type,
                'position' => $match[1],
209 210 211 212
            );
        }
    }
    
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
    /**
     * Gets the literal for a given token.
     *
     * @param integer $token
     * @return string
     */
    public function getLiteral($token)
    {
        $className = get_class($this);
        $reflClass = new \ReflectionClass($className);
        $constants = $reflClass->getConstants();
        
        foreach ($constants as $name => $value) {
            if ($value === $token) {
                return $className . '::' . $name;
            }
        }
        
        return $token;
    }
    
234
    /**
235
     * Lexical catchable patterns.
236 237 238 239 240 241
     *
     * @return array
     */
    abstract protected function getCatchablePatterns();
    
    /**
242
     * Lexical non-catchable patterns.
243 244 245 246 247 248 249 250 251 252 253 254 255
     *
     * @return array
     */
    abstract protected function getNonCatchablePatterns();
    
    /**
     * Retrieve token type. Also processes the token value if necessary.
     *
     * @param string $value
     * @return integer
     */
    abstract protected function _getType(&$value);
}