# Source for file Tokenizer.php

Documentation is available at Tokenizer.php

1. <?php
2. /*
3. * $Id: From.php 1080 2007-02-10 18:17:08Z romanb $
4. *
5. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
6. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
7. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
8. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
9. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
10. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
11. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
12. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
13. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
14. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
15. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
16. *
17. * This software consists of voluntary contributions made by many individuals
18. * and is licensed under the LGPL. For more information, see
19. * <http://www.phpdoctrine.com>.
20. */
22. /**
23. * Doctrine_Tokenizer
24. *
25. * @package Doctrine
26. * @license http://www.opensource.org/licenses/lgpl-license.php LGPL
27. * @category Object Relational Mapping
28. * @link www.phpdoctrine.com
29. * @since 1.0
30. * @version $Revision: 1080 $
31. * @author Konsta Vesterinen <kvesteri@cc.hut.fi>
32. */
33. class Doctrine_Tokenizer
34. {
35. public function __construct()
36. {
38. }
39. public function tokenize()
40. {
42. }
43. /**
44. * trims brackets
45. *
46. * @param string $str
47. * @param string $e1 the first bracket, usually '('
48. * @param string $e2 the second bracket, usually ')'
49. */
50. public static function bracketTrim($str, $e1 = '(', $e2 = ')')
51. {
52. if (substr($str, 0, 1) === $e1 && substr($str, -1) === $e2) {
53. return substr($str, 1, -1);
54. } else {
55. return $str;
56. }
57. }
58. /**
59. * bracketExplode
60. *
61. * example:
62. *
63. * parameters:
64. * $str = (age < 20 AND age > 18) AND email LIKE 'John@example.com'
65. * $d = ' AND '
66. * $e1 = '('
67. * $e2 = ')'
68. *
69. * would return an array:
70. * array("(age < 20 AND age > 18)",
71. * "email LIKE 'John@example.com'")
72. *
73. * @param string $str
74. * @param string $d the delimeter which explodes the string
75. * @param string $e1 the first bracket, usually '('
76. * @param string $e2 the second bracket, usually ')'
77. *
78. */
79. public static function bracketExplode($str, $d = ' ', $e1 = '(', $e2 = ')')
80. {
81. if(is_array($d)) {
82. $a = preg_split('/('.implode('|', $d).')/', $str);
83. $d = stripslashes($d[0]);
84. } else {
85. $a = explode($d, $str);
86. }
88. $i = 0;
89. $term = array();
90. foreach($a as $key=>$val) {
91. if (empty($term[$i])) {
92. $term[$i] = trim($val);
93. $s1 = substr_count($term[$i], $e1);
94. $s2 = substr_count($term[$i], $e2);
96. if($s1 == $s2) {
97. $i++;
98. }
99. } else {
100. $term[$i] .= $d . trim($val);
101. $c1 = substr_count($term[$i], $e1);
102. $c2 = substr_count($term[$i], $e2);
104. if($c1 == $c2) {
105. $i++;
106. }
107. }
108. }
109. return $term;
110. }
111. /**
112. * quoteExplode
113. *
114. * example:
115. *
116. * parameters:
117. * $str = email LIKE 'John@example.com'
118. * $d = ' AND '
119. *
120. * would return an array:
121. * array("email", "LIKE", "'John@example.com'")
122. *
123. * @param string $str
124. * @param string $d the delimeter which explodes the string
125. */
126. public static function quoteExplode($str, $d = ' ')
127. {
128. if (is_array($d)) {
129. $a = preg_split('/('.implode('|', $d).')/', $str);
130. $d = stripslashes($d[0]);
131. } else {
132. $a = explode($d, $str);
133. }
135. $i = 0;
136. $term = array();
137. foreach ($a as $key => $val) {
138. if (empty($term[$i])) {
139. $term[$i] = trim($val);
141. if ( ! (substr_count($term[$i], "'") & 1)) {
142. $i++;
143. }
144. } else {
145. $term[$i] .= $d . trim($val);
147. if ( ! (substr_count($term[$i], "'") & 1)) {
148. $i++;
149. }
150. }
151. }
152. return $term;
153. }
154. /**
155. * sqlExplode
156. *
157. * explodes a string into array using custom brackets and
158. * quote delimeters
159. *
160. *
161. * example:
162. *
163. * parameters:
164. * $str = "(age < 20 AND age > 18) AND name LIKE 'John Doe'"
165. * $d = ' '
166. * $e1 = '('
167. * $e2 = ')'
168. *
169. * would return an array:
170. * array('(age < 20 AND age > 18)',
171. * 'name',
172. * 'LIKE',
173. * 'John Doe')
174. *
175. * @param string $str
176. * @param string $d the delimeter which explodes the string
177. * @param string $e1 the first bracket, usually '('
178. * @param string $e2 the second bracket, usually ')'
179. *
180. * @return array
181. */
182. public static function sqlExplode($str, $d = ' ', $e1 = '(', $e2 = ')')
183. {
184. if ($d == ' ') {
185. $d = array(' ', '\s');
186. }
187. if (is_array($d)) {
188. $d = array_map('preg_quote', $d);
190. if (in_array(' ', $d)) {
191. $d[] = '\s';
192. }
194. $split = '§(' . implode('|', $d) . ')§';
196. $str = preg_split($split, $str);
197. $d = stripslashes($d[0]);
198. } else {
199. $str = explode($d, $str);
200. }
202. $i = 0;
203. $term = array();
205. foreach ($str as $key => $val) {
206. if (empty($term[$i])) {
207. $term[$i] = trim($val);
209. $s1 = substr_count($term[$i], $e1);
210. $s2 = substr_count($term[$i], $e2);
212. if (strpos($term[$i], '(') !== false) {
213. if($s1 == $s2) {
214. $i++;
215. }
216. } else {
217. if ( ! (substr_count($term[$i], "'") & 1) &&
218. ! (substr_count($term[$i], "\"") & 1)) {
219. $i++;
220. }
221. }
222. } else {
223. $term[$i] .= $d . trim($val);
224. $c1 = substr_count($term[$i], $e1);
225. $c2 = substr_count($term[$i], $e2);
227. if (strpos($term[$i], '(') !== false) {
228. if($c1 == $c2) {
229. $i++;
230. }
231. } else {
232. if ( ! (substr_count($term[$i], "'") & 1) &&
233. ! (substr_count($term[$i], "\"") & 1)) {
234. $i++;
235. }
236. }
237. }
238. }
239. return $term;
240. }
241. /**
242. * clauseExplode
243. *
244. * explodes a string into array using custom brackets and
245. * quote delimeters
246. *
247. *
248. * example:
249. *
250. * parameters:
251. * $str = "(age < 20 AND age > 18) AND name LIKE 'John Doe'"
252. * $d = ' '
253. * $e1 = '('
254. * $e2 = ')'
255. *
256. * would return an array:
257. * array('(age < 20 AND age > 18)',
258. * 'name',
259. * 'LIKE',
260. * 'John Doe')
261. *
262. * @param string $str
263. * @param string $d the delimeter which explodes the string
264. * @param string $e1 the first bracket, usually '('
265. * @param string $e2 the second bracket, usually ')'
266. *
267. * @return array
268. */
269. public static function clauseExplode($str, array $d, $e1 = '(', $e2 = ')')
270. {
271. if (is_array($d)) {
272. $d = array_map('preg_quote', $d);
274. if (in_array(' ', $d)) {
275. $d[] = '\s';
276. }
278. $split = '§(' . implode('|', $d) . ')§';
280. $str = preg_split($split, $str, -1, PREG_SPLIT_DELIM_CAPTURE);
281. }
283. $i = 0;
284. $term = array();
286. foreach ($str as $key => $val) {
287. if ($key & 1) {
288. if (isset($term[($i - 1)]) && ! is_array($term[($i - 1)])) {
289. $term[($i - 1)] = array($term[($i - 1)], $val);
290. }
291. continue;
292. }
293. if (empty($term[$i])) {
294. $term[$i] = $val;
295. } else {
296. $term[$i] .= $str[($key - 1)] . $val;
297. }
299. $c1 = substr_count($term[$i], $e1);
300. $c2 = substr_count($term[$i], $e2);
302. if (strpos($term[$i], '(') !== false) {
303. if($c1 == $c2) {
304. $i++;
305. }
306. } else {
307. if ( ! (substr_count($term[$i], "'") & 1) &&
308. ! (substr_count($term[$i], "\"") & 1)) {
309. $i++;
310. }
311. }
312. }
313. $term[$i - 1] = array($term[$i - 1], '');
315. return $term;
316. }
317. }