prepared the tokenizer for getting a state and being a full-fledged object

lib/Doctrine/Expression.php

@@ -33,8 +33,8 @@ Doctrine::autoload('Doctrine_Connection_Module');
 class Doctrine_Expression
 {
     protected $_expression;
-    
     protected $_conn;
+    protected $_tokenizer;
 
     /**
      * Create an expression 
@@ -45,8 +45,8 @@ class Doctrine_Expression
      */
     public function __construct($expr, $conn = null)
     {
+        $this->_tokenizer = new Doctrine_Query_Tokenizer();
         $this->setExpression($expr);
-        
         if ($conn !== null) {
             $this->_conn = $conn;
         }
@@ -97,7 +97,7 @@ class Doctrine_Expression
         $argStr = substr($expr, ($pos + 1), -1);
 
         // parse args
-        foreach (Doctrine_Tokenizer::bracketExplode($argStr, ',') as $arg) {
+        foreach ($this->_tokenizer->bracketExplode($argStr, ',') as $arg) {
            $args[] = $this->parseClause($arg);
         }
 
@@ -112,7 +112,7 @@ class Doctrine_Expression
      */
     public function parseClause($clause)
     {
-        $e = Doctrine_Tokenizer::bracketExplode($clause, ' ');
+        $e = $this->_tokenizer->bracketExplode($clause, ' ');
 
         foreach ($e as $k => $expr) {
             $e[$k] = $this->parseExpression($expr);

lib/Doctrine/Hook/Parser/Complex.php

@@ -32,6 +32,16 @@ Doctrine::autoload('Doctrine_Hook_Parser');
  */
 abstract class Doctrine_Hook_Parser_Complex extends Doctrine_Hook_Parser
 {
+    protected $_tokenizer;
+    
+    /**
+     * Constructor.
+     */
+    public function __construct()
+    {
+        $this->_tokenizer = new Doctrine_Query_Tokenizer();
+    }
+    
     /**
      * parse
      * Parses given field and field value to DQL condition
@@ -59,7 +69,7 @@ abstract class Doctrine_Hook_Parser_Complex extends Doctrine_Hook_Parser
      */
     public function parseClause($alias, $field, $value)
     {
-        $parts = Doctrine_Tokenizer::quoteExplode($value, ' AND ');
+        $parts = $this->_tokenizer->quoteExplode($value, ' AND ');
 
         if (count($parts) > 1) {
             $ret = array();
@@ -69,7 +79,7 @@ abstract class Doctrine_Hook_Parser_Complex extends Doctrine_Hook_Parser
 
             $r = implode(' AND ', $ret);
         } else {
-            $parts = Doctrine_Tokenizer::quoteExplode($value, ' OR ');
+            $parts = $this->_tokenizer->quoteExplode($value, ' OR ');
             if (count($parts) > 1) {
                 $ret = array();
                 foreach ($parts as $part) {

lib/Doctrine/Hook/WordLike.php

@@ -47,7 +47,7 @@ class Doctrine_Hook_WordLike extends Doctrine_Hook_Parser_Complex
     public function parseSingle($alias, $field, $value)
     {
         if (strpos($value, "'") !== false) {
-            $value = Doctrine_Tokenizer::bracketTrim($value, "'", "'");
+            $value = $this->_tokenizer->bracketTrim($value, "'", "'");
         
             $a[]   = $alias . '.' . $field . ' LIKE ?';
             $this->params[] = '%' . $value . '%';

lib/Doctrine/Query.php

@@ -1063,7 +1063,7 @@ class Doctrine_Query extends Doctrine_Query_Abstract implements Countable, Seria
                 }
 
                 if (isset($this->_pendingJoinConditions[$k])) {
-                    $parser = new Doctrine_Query_JoinCondition($this);
+                    $parser = new Doctrine_Query_JoinCondition($this, $this->_tokenizer);
 
                     if (strpos($part, ' ON ') !== false) {
                         $part .= ' AND ';

lib/Doctrine/Query/Abstract.php

@@ -229,12 +229,17 @@ abstract class Doctrine_Query_Abstract
      */
     protected $_enumParams = array();
     
+    /**
+     * @var boolean
+     */
     protected $_isLimitSubqueryUsed = false;
     
     
     /**
+     * Constructor.
      *
-     *
+     * @param Doctrine_Connection  The connection object the query will use.
+     * @param Doctrine_Hydrator_Abstract  The hydrator that will be used for generating result sets.
      */
     public function __construct(Doctrine_Connection $connection = null,
             Doctrine_Hydrator_Abstract $hydrator = null)
@@ -1685,7 +1690,7 @@ abstract class Doctrine_Query_Abstract
                 throw new Doctrine_Query_Exception('Unknown parser ' . $name);
             }
 
-            $this->_parsers[$name] = new $class($this);
+            $this->_parsers[$name] = new $class($this, $this->_tokenizer);
         }
 
         return $this->_parsers[$name];

lib/Doctrine/Query/Check.php

@@ -42,6 +42,8 @@ class Doctrine_Query_Check
      *                                      parsed from the given dql CHECK definition
      */
     protected $sql;
+    
+    protected $_tokenizer;
 
     /**
      * @param Doctrine_Table|string $table  Doctrine_Table object
@@ -54,6 +56,7 @@ class Doctrine_Query_Check
                         ->getTable($table);
         }
         $this->table = $table;
+        $this->_tokenizer = new Doctrine_Query_Tokenizer();
     }
 
     /**
@@ -88,7 +91,7 @@ class Doctrine_Query_Check
      */
     public function parseClause($dql)
     {
-        $parts = Doctrine_Tokenizer::sqlExplode($dql, ' AND ');
+        $parts = $this->_tokenizer->sqlExplode($dql, ' AND ');
 
         if (count($parts) > 1) {
             $ret = array();
@@ -98,7 +101,7 @@ class Doctrine_Query_Check
 
             $r = implode(' AND ', $ret);
         } else {
-            $parts = Doctrine_Tokenizer::quoteExplode($dql, ' OR ');
+            $parts = $this->_tokenizer->quoteExplode($dql, ' OR ');
             if (count($parts) > 1) {
                 $ret = array();
                 foreach ($parts as $part) {
@@ -113,6 +116,7 @@ class Doctrine_Query_Check
         }
         return '(' . $r . ')';
     }
+    
     public function parseSingle($part)
     {
         $e = explode(' ', $part);

lib/Doctrine/Query/Condition.php

@@ -43,22 +43,22 @@ abstract class Doctrine_Query_Condition extends Doctrine_Query_Part
     {
         $tmp = trim($str);
 
-        $parts = Doctrine_Tokenizer::bracketExplode($str, array(' \&\& ', ' AND '), '(', ')');
+        $parts = $this->_tokenizer->bracketExplode($str, array(' \&\& ', ' AND '), '(', ')');
 
         if (count($parts) > 1) {
             $ret = array();
             foreach ($parts as $part) {
-                $part = Doctrine_Tokenizer::bracketTrim($part, '(', ')');
+                $part = $this->_tokenizer->bracketTrim($part, '(', ')');
                 $ret[] = $this->parse($part);
             }
             $r = implode(' AND ', $ret);
         } else {
 
-            $parts = Doctrine_Tokenizer::bracketExplode($str, array(' \|\| ', ' OR '), '(', ')');
+            $parts = $this->_tokenizer->bracketExplode($str, array(' \|\| ', ' OR '), '(', ')');
             if (count($parts) > 1) {
                 $ret = array();
                 foreach ($parts as $part) {
-                    $part = Doctrine_Tokenizer::bracketTrim($part, '(', ')');
+                    $part = $this->_tokenizer->bracketTrim($part, '(', ')');
                     $ret[] = $this->parse($part);
                 }
                 $r = implode(' OR ', $ret);

lib/Doctrine/Query/From.php

@@ -42,7 +42,7 @@ class Doctrine_Query_From extends Doctrine_Query_Part
     public function parse($str)
     {        
         $str = trim($str);
-        $parts = Doctrine_Tokenizer::bracketExplode($str, 'JOIN');
+        $parts = $this->_tokenizer->bracketExplode($str, 'JOIN');
 
         $operator = false;
 
@@ -70,7 +70,7 @@ class Doctrine_Query_From extends Doctrine_Query_Part
             }
             $part = implode(' ', $e);
 
-            foreach (Doctrine_Tokenizer::bracketExplode($part, ',') as $reference) {
+            foreach ($this->_tokenizer->bracketExplode($part, ',') as $reference) {
                 $reference = trim($reference);
                 $e = explode(' ', $reference);
                 $e2 = explode('.', $e[0]);

lib/Doctrine/Query/Having.php

@@ -47,7 +47,7 @@ class Doctrine_Query_Having extends Doctrine_Query_Condition
 
             $name   = substr($func, 0, $pos);
             $func   = substr($func, ($pos + 1), -1);
-            $params = Doctrine_Tokenizer::bracketExplode($func, ',', '(', ')');
+            $params = $this->_tokenizer->bracketExplode($func, ',', '(', ')');
 
             foreach ($params as $k => $param) {
                 $params[$k] = $this->parseAggregateFunction($param);
@@ -88,7 +88,7 @@ class Doctrine_Query_Having extends Doctrine_Query_Condition
      */
     final public function load($having)
     {
-        $tokens = Doctrine_Tokenizer::bracketExplode($having, ' ', '(', ')');
+        $tokens = $this->_tokenizer->bracketExplode($having, ' ', '(', ')');
         $part = $this->parseAggregateFunction(array_shift($tokens));
         $operator  = array_shift($tokens);
         $value     = implode(' ', $tokens);

lib/Doctrine/Query/JoinCondition.php

@@ -36,7 +36,7 @@ class Doctrine_Query_JoinCondition extends Doctrine_Query_Condition
     {
         $condition = trim($condition);
 
-        $e         = Doctrine_Tokenizer::sqlExplode($condition);
+        $e = $this->_tokenizer->sqlExplode($condition);
 
         if (count($e) > 2) {
             $a         = explode('.', $e[0]);
@@ -54,7 +54,7 @@ class Doctrine_Query_JoinCondition extends Doctrine_Query_Condition
 
             if (substr($value, 0, 1) == '(') {
                 // trim brackets
-                $trimmed   = Doctrine_Tokenizer::bracketTrim($value);
+                $trimmed   = $this->_tokenizer->bracketTrim($value);
 
                 if (substr($trimmed, 0, 4) == 'FROM' || substr($trimmed, 0, 6) == 'SELECT') {
                     // subquery found
@@ -64,7 +64,7 @@ class Doctrine_Query_JoinCondition extends Doctrine_Query_Condition
                     $value = '(' . substr($trimmed, 4) . ')';
                 } else {
                     // simple in expression found
-                    $e     = Doctrine_Tokenizer::sqlExplode($trimmed, ',');
+                    $e     = $this->_tokenizer->sqlExplode($trimmed, ',');
 
                     $value = array();
                     foreach ($e as $part) {

lib/Doctrine/Query/Part.php

@@ -36,13 +36,19 @@ abstract class Doctrine_Query_Part
      * @var Doctrine_Query $query           the query object associated with this parser
      */
     protected $query;
+    
+    protected $_tokenizer;
 
     /**
      * @param Doctrine_Query $query         the query object associated with this parser
      */
-    public function __construct($query)
+    public function __construct($query, Doctrine_Query_Tokenizer $tokenizer = null)
     {
         $this->query = $query;
+        if ( ! $tokenizer) {
+            $tokenizer = new Doctrine_Query_Tokenizer();
+        }
+        $this->_tokenizer = $tokenizer;
     }
 
     /**

lib/Doctrine/Query/Set.php

@@ -34,7 +34,7 @@ class Doctrine_Query_Set extends Doctrine_Query_Part
 {
     public function parse($dql)
     {
-    	$terms = Doctrine_Tokenizer::sqlExplode($dql, ' ');
+    	$terms = $this->_tokenizer->sqlExplode($dql, ' ');
     	
     	foreach ($terms as $term) {
 

lib/Doctrine/Query/Tokenizer.php

@@ -28,6 +28,10 @@
  * @link        www.phpdoctrine.com
  * @since       1.0
  * @version     $Revision$
+ * @todo Give the tokenizer state, make it better work together with Doctrine_Query and maybe
+ *       take out commonly used string manipulation methods
+ *       into a stateless StringUtil? class. This tokenizer should be concerned with tokenizing
+ *       DQL strings.
  */
 class Doctrine_Query_Tokenizer 
 {
@@ -54,7 +58,7 @@ class Doctrine_Query_Tokenizer
     public function tokenizeQuery($query)
     {
         $parts = array();
-        $tokens = Doctrine_Tokenizer::sqlExplode($query, ' ');
+        $tokens = $this->sqlExplode($query, ' ');
 
         foreach ($tokens as $index => $token) {
             $token = strtolower(trim($token));

lib/Doctrine/Query/Where.php

@@ -34,9 +34,9 @@ class Doctrine_Query_Where extends Doctrine_Query_Condition
 {
     public function load($where) 
     {
-        $where = Doctrine_Tokenizer::bracketTrim(trim($where));
+        $where = $this->_tokenizer->bracketTrim(trim($where));
         $conn  = $this->query->getConnection();
-        $terms = Doctrine_Tokenizer::sqlExplode($where);  
+        $terms = $this->_tokenizer->sqlExplode($where);  
 
         if (count($terms) > 1) {
             if (substr($where, 0, 6) == 'EXISTS') {
@@ -47,7 +47,7 @@ class Doctrine_Query_Where extends Doctrine_Query_Condition
         }
 
         if (count($terms) < 3) {
-            $terms = Doctrine_Tokenizer::sqlExplode($where, array('=', '<', '<>', '>', '!='));
+            $terms = $this->_tokenizer->sqlExplode($where, array('=', '<', '<>', '>', '!='));
         }
 
         if (count($terms) > 1) {
@@ -90,7 +90,7 @@ class Doctrine_Query_Where extends Doctrine_Query_Condition
     {
         if (substr($value, 0, 1) == '(') {
             // trim brackets
-            $trimmed   = Doctrine_Tokenizer::bracketTrim($value);
+            $trimmed = $this->_tokenizer->bracketTrim($value);
 
             if (substr($trimmed, 0, 4) == 'FROM' ||
                 substr($trimmed, 0, 6) == 'SELECT') {
@@ -103,7 +103,7 @@ class Doctrine_Query_Where extends Doctrine_Query_Condition
                 $value = '(' . substr($trimmed, 4) . ')';
             } else {
                 // simple in expression found
-                $e = Doctrine_Tokenizer::sqlExplode($trimmed, ',');
+                $e = $this->_tokenizer->sqlExplode($trimmed, ',');
 
                 $value = array();
 
@@ -123,7 +123,7 @@ class Doctrine_Query_Where extends Doctrine_Query_Condition
 
                 $value = '(' . implode(', ', $value) . ')';
             }
-        } elseif (substr($value, 0, 1) == ':' || $value === '?') {
+        } else if (substr($value, 0, 1) == ':' || $value === '?') {
             // placeholder found
             if (isset($table) && isset($field) && $table->getTypeOf($field) == 'enum') {
                 $this->query->addEnumParam($value, $table, $field);
@@ -163,7 +163,7 @@ class Doctrine_Query_Where extends Doctrine_Query_Condition
             throw new Doctrine_Query_Exception('Unknown expression, expected a subquery with () -marks');
         }
 
-        $sub = Doctrine_Tokenizer::bracketTrim(substr($where, $pos));
+        $sub = $this->_tokenizer->bracketTrim(substr($where, $pos));
 
         return $operator . ' (' . $this->query->createSubquery()->parseQuery($sub, false)->getQuery() . ')';
     }

lib/Doctrine/Search/Query.php

@@ -44,6 +44,7 @@ class Doctrine_Search_Query
     
     protected $_words = array();
     
+    protected $_tokenizer;
 
     protected $_condition;
 
@@ -60,6 +61,7 @@ class Doctrine_Search_Query
             }
         }
 
+        $this->_tokenizer = new Doctrine_Query_Tokenizer();
         $this->_table = $table;
 
         $foreignId = current(array_diff($this->_table->getColumnNames(), array('keyword', 'field', 'position')));
@@ -97,7 +99,7 @@ class Doctrine_Search_Query
 
     public function parseClause($originalClause, $recursive = false)
     {
-        $clause = Doctrine_Tokenizer::bracketTrim($originalClause);
+        $clause = $this->_tokenizer->bracketTrim($originalClause);
         
         $brackets = false;
 
@@ -107,7 +109,7 @@ class Doctrine_Search_Query
 
         $foreignId = current(array_diff($this->_table->getColumnNames(), array('keyword', 'field', 'position')));
         
-        $terms = Doctrine_Tokenizer::sqlExplode($clause, ' OR ', '(', ')');
+        $terms = $this->_tokenizer->sqlExplode($clause, ' OR ', '(', ')');
 
         $ret = array();
 
@@ -130,7 +132,7 @@ class Doctrine_Search_Query
                 $brackets = false;
             }
         } else {
-            $terms = Doctrine_Tokenizer::sqlExplode($clause, ' ', '(', ')');
+            $terms = $this->_tokenizer->sqlExplode($clause, ' ', '(', ')');
             
             if (count($terms) === 1 && ! $recursive) {
                 $return = $this->parseTerm($clause);
@@ -170,7 +172,7 @@ class Doctrine_Search_Query
         if (strpos($term, '(') !== false) {
             return true;
         } else {
-            $terms = Doctrine_Tokenizer::quoteExplode($term);
+            $terms = $this->_tokenizer->quoteExplode($term);
             
             return (count($terms) > 1);
         }
@@ -185,7 +187,7 @@ class Doctrine_Search_Query
         } else {
             $term = trim($term, "' ");
 
-            $terms = Doctrine_Tokenizer::quoteExplode($term);
+            $terms = $this->_tokenizer->quoteExplode($term);
             $where = $this->parseWord($terms[0]);
 
             foreach ($terms as $k => $word) {

lib/Doctrine/Tokenizer.php

-<?php
-/*
- *  $Id: From.php 1080 2007-02-10 18:17:08Z romanb $
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * This software consists of voluntary contributions made by many individuals
- * and is licensed under the LGPL. For more information, see
- * <http://www.phpdoctrine.com>.
- */
-
-/**
- * Doctrine_Tokenizer
- *
- * @package     Doctrine
- * @subpackage  Tokenizer
- * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
- * @link        www.phpdoctrine.com
- * @since       1.0
- * @version     $Revision: 1080 $
- * @author      Konsta Vesterinen <kvesteri@cc.hut.fi>
- */
-class Doctrine_Tokenizer 
-{
-    /**
-     * trims brackets
-     *
-     * @param string $str
-     * @param string $e1        the first bracket, usually '('
-     * @param string $e2        the second bracket, usually ')'
-     */
-    public static function bracketTrim($str, $e1 = '(', $e2 = ')')
-    {
-        if (substr($str, 0, 1) === $e1 && substr($str, -1) === $e2) {
-            return substr($str, 1, -1);
-        } else {
-            return $str;
-        }
-    }
-
-    /**
-     * bracketExplode
-     *
-     * example:
-     *
-     * parameters:
-     *      $str = (age < 20 AND age > 18) AND email LIKE 'John@example.com'
-     *      $d = ' AND '
-     *      $e1 = '('
-     *      $e2 = ')'
-     *
-     * would return an array:
-     *      array("(age < 20 AND age > 18)",
-     *            "email LIKE 'John@example.com'")
-     *
-     * @param string $str
-     * @param string $d         the delimeter which explodes the string
-     * @param string $e1        the first bracket, usually '('
-     * @param string $e2        the second bracket, usually ')'
-     *
-     */
-    public static function bracketExplode($str, $d = ' ', $e1 = '(', $e2 = ')')
-    {
-        if (is_array($d)) {
-            $a = preg_split('/('.implode('|', $d).')/', $str);
-            $d = stripslashes($d[0]);
-        } else {
-            $a = explode($d, $str);
-        }
-
-        $i = 0;
-        $term = array();
-        foreach($a as $key=>$val) {
-            if (empty($term[$i])) {
-                $term[$i] = trim($val);
-                $s1 = substr_count($term[$i], $e1);
-                $s2 = substr_count($term[$i], $e2);
-                
-                if ($s1 == $s2) {
-                    $i++;
-                }
-            } else {
-                $term[$i] .= $d . trim($val);
-                $c1 = substr_count($term[$i], $e1);
-                $c2 = substr_count($term[$i], $e2);
-                
-                if ($c1 == $c2) { 
-                    $i++;
-                }
-            }
-        }
-        return $term;
-    }
-
-    /**
-     * quoteExplode
-     *
-     * example:
-     *
-     * parameters:
-     *      $str = email LIKE 'John@example.com'
-     *      $d = ' LIKE '
-     *
-     * would return an array:
-     *      array("email", "LIKE", "'John@example.com'")
-     *
-     * @param string $str
-     * @param string $d         the delimeter which explodes the string
-     */
-    public static function quoteExplode($str, $d = ' ')
-    {
-        if (is_array($d)) {
-            $a = preg_split('/('.implode('|', $d).')/', $str);
-            $d = stripslashes($d[0]);
-        } else {
-            $a = explode($d, $str);
-        }
-
-        $i = 0;
-        $term = array();
-        foreach ($a as $key => $val) {
-            if (empty($term[$i])) {
-                $term[$i] = trim($val);
-
-                if ( ! (substr_count($term[$i], "'") & 1)) {
-                    $i++;
-                }
-            } else {
-                $term[$i] .= $d . trim($val);
-
-                if ( ! (substr_count($term[$i], "'") & 1)) {
-                    $i++;
-                }
-            }
-        }
-        return $term;
-    }
-
-    /**
-     * sqlExplode
-     *
-     * explodes a string into array using custom brackets and
-     * quote delimeters
-     *
-     *
-     * example:
-     *
-     * parameters:
-     *      $str = "(age < 20 AND age > 18) AND name LIKE 'John Doe'"
-     *      $d   = ' '
-     *      $e1  = '('
-     *      $e2  = ')'
-     *
-     * would return an array:
-     *      array('(age < 20 AND age > 18)',
-     *            'name',
-     *            'LIKE',
-     *            'John Doe')
-     *
-     * @param string $str
-     * @param string $d         the delimeter which explodes the string
-     * @param string $e1        the first bracket, usually '('
-     * @param string $e2        the second bracket, usually ')'
-     *
-     * @return array
-     */
-    public static function sqlExplode($str, $d = ' ', $e1 = '(', $e2 = ')')
-    {
-        if ($d == ' ') {
-            $d = array(' ', '\s');
-        }
-        if (is_array($d)) {
-            $d = array_map('preg_quote', $d);
-
-            if (in_array(' ', $d)) {
-                $d[] = '\s';
-            }
-
-            $split = '§(' . implode('|', $d) . ')§';
-
-            $str = preg_split($split, $str);
-            $d = stripslashes($d[0]);
-        } else {
-            $str = explode($d, $str);
-        }
-
-        $i = 0;
-        $term = array();
-
-        foreach ($str as $key => $val) {
-            if (empty($term[$i])) {
-                $term[$i] = trim($val);
-
-                $s1 = substr_count($term[$i], $e1);
-                $s2 = substr_count($term[$i], $e2);
-
-                if (strpos($term[$i], '(') !== false) {
-                    if ($s1 == $s2) {
-                        $i++;
-                    }
-                } else {
-                    if ( ! (substr_count($term[$i], "'") & 1) &&
-                         ! (substr_count($term[$i], "\"") & 1)) {
-                        $i++;
-                    }
-                }
-            } else {
-                $term[$i] .= $d . trim($val);
-                $c1 = substr_count($term[$i], $e1);
-                $c2 = substr_count($term[$i], $e2);
-
-                if (strpos($term[$i], '(') !== false) {
-                    if ($c1 == $c2) {
-                        $i++;
-                    }
-                } else {
-                    if ( ! (substr_count($term[$i], "'") & 1) &&
-                         ! (substr_count($term[$i], "\"") & 1)) {
-                        $i++;
-                    }
-                }
-            }
-        }
-        return $term;
-    }
-
-    /**
-     * clauseExplode
-     *
-     * explodes a string into array using custom brackets and
-     * quote delimeters
-     *
-     *
-     * example:
-     *
-     * parameters:
-     *      $str = "(age < 20 AND age > 18) AND name LIKE 'John Doe'"
-     *      $d   = ' '
-     *      $e1  = '('
-     *      $e2  = ')'
-     *
-     * would return an array:
-     *      array('(age < 20 AND age > 18)',
-     *            'name',
-     *            'LIKE',
-     *            'John Doe')
-     *
-     * @param string $str
-     * @param string $d         the delimeter which explodes the string
-     * @param string $e1        the first bracket, usually '('
-     * @param string $e2        the second bracket, usually ')'
-     *
-     * @return array
-     */
-    public static function clauseExplode($str, array $d, $e1 = '(', $e2 = ')')
-    {
-        if (is_array($d)) {
-            $d = array_map('preg_quote', $d);
-
-            if (in_array(' ', $d)) {
-                $d[] = '\s';
-            }
-
-            $split = '§(' . implode('|', $d) . ')§';
-
-            $str = preg_split($split, $str, -1, PREG_SPLIT_DELIM_CAPTURE);
-        }
-
-        $i = 0;
-        $term = array();
-
-        foreach ($str as $key => $val) {
-            if ($key & 1) {
-                if (isset($term[($i - 1)]) && ! is_array($term[($i - 1)])) {
-                    $term[($i - 1)] = array($term[($i - 1)], $val);
-                }
-                continue;
-            }
-            if (empty($term[$i])) {
-                $term[$i] = $val;
-            } else {
-                $term[$i] .= $str[($key - 1)] . $val;
-            }
-
-            $c1 = substr_count($term[$i], $e1);
-            $c2 = substr_count($term[$i], $e2);
-
-            if (strpos($term[$i], '(') !== false) {
-                if ($c1 == $c2) {
-                    $i++;
-                }
-            } else {
-                if ( ! (substr_count($term[$i], "'") & 1) &&
-                     ! (substr_count($term[$i], "\"") & 1)) {
-                    $i++;
-                }
-            }
-        }
-        $term[$i - 1] = array($term[$i - 1], '');
-
-        return $term;
-    }
-}

tests/Query/ConditionTestCase.php

@@ -34,11 +34,13 @@ class Doctrine_Query_Condition_TestCase extends Doctrine_UnitTestCase
 {
     public function prepareData() { }
     public function prepareTables() { }
-
+    
+    /** @todo belongs in TokenizerTestCase? */
     public function testBracktExplode() 
     {
+        $tokenizer = new Doctrine_Query_Tokenizer();
         $str   = "item OR item || item";
-        $parts = Doctrine_Tokenizer::bracketExplode($str, array(' \|\| ', ' OR '), "(", ")");
+        $parts = $tokenizer->bracketExplode($str, array(' \|\| ', ' OR '), "(", ")");
 
         $this->assertEqual($parts, array('item','item','item'));
 

tests/TokenizerTestCase.php

@@ -40,72 +40,75 @@ class Doctrine_Tokenizer_TestCase extends Doctrine_UnitTestCase
 
     public function testSqlExplode()
     {
+        $tokenizer = new Doctrine_Query_Tokenizer();
+        
         $str = "word1 word2 word3";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
 
         $this->assertEqual($a, array("word1", "word2", "word3"));
 
         $str = "word1 (word2 word3)";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "(word2 word3)"));
 
         $str = "word1 'word2 word3'";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "'word2 word3'"));
 
         $str = "word1 'word2 word3'";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "'word2 word3'"));
 
         $str = "word1 \"word2 word3\"";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "\"word2 word3\""));
 
         $str = "word1 ((word2) word3)";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "((word2) word3)"));
 
         $str = "word1 ( (word2) 'word3')";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "( (word2) 'word3')"));
 
         $str = "word1 ( \"(word2) 'word3')";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "( \"(word2) 'word3')"));
 
         $str = "word1 ( ''(word2) 'word3')";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "( ''(word2) 'word3')"));
 
         $str = "word1 ( '()()'(word2) 'word3')";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "( '()()'(word2) 'word3')"));
 
         $str = "word1 'word2)() word3'";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "'word2)() word3'"));
 
         $str = "word1 (word2() word3)";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "(word2() word3)"));
 
         $str = "word1 \"word2)() word3\"";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("word1", "\"word2)() word3\""));
 
         $str = "something (subquery '')";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("something", "(subquery '')"));
 
         $str = "something ((  ))";
-        $a   = Doctrine_Tokenizer::sqlExplode($str);
+        $a   = $tokenizer->sqlExplode($str);
         $this->assertEqual($a, array("something", "((  ))"));
     }
 
     public function testSqlExplode2()
     {
+        $tokenizer = new Doctrine_Query_Tokenizer();
         $str = 'rdbms (dbal OR database)';
-        $a   = Doctrine_Tokenizer::sqlExplode($str, ' OR ');
+        $a   = $tokenizer->sqlExplode($str, ' OR ');
 
         $this->assertEqual($a, array('rdbms (dbal OR database)'));
     }
@@ -113,7 +116,8 @@ class Doctrine_Tokenizer_TestCase extends Doctrine_UnitTestCase
 
     public function testQuoteExplodedShouldQuoteArray()
     {
-        $term = Doctrine_Tokenizer::quoteExplode("test", array("'test'", "test2"));
+        $tokenizer = new Doctrine_Query_Tokenizer();
+        $term = $tokenizer->quoteExplode("test", array("'test'", "test2"));
         $this->assertEqual($term[0], "test");
     }
 }