Mysql bulk insert support => huge performance increase for mysql session

classes/DQL/Parser.class.php

@@ -725,7 +725,7 @@ class Doctrine_DQL_Parser {
             $reference = implode(".",$a);
             $objTable   = $this->session->getTable(end($a));
             $where     = $objTable->getTableName().".".$field." ".$operator." ".$value;
-            if(count($a) > 1 AND isset($a[1])) {
+            if(count($a) > 1 && isset($a[1])) {
                 $root = $a[0];
                 $fk = $this->tnames[$root]->getForeignKey($a[1]);
                 if($fk instanceof Doctrine_Association) {

classes/Session.class.php

@@ -29,43 +29,45 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
      * @var $dbh                            the database handle
      */
     private $dbh;
+    /**
+     * @see Doctrine_Session::STATE_* constants
+     * @var boolean $state                  the current state of the session
+     */
+    private $state          = 0;
+    /**
+     * @var integer $transaction_level      the nesting level of transactions, used by transaction methods
+     */
+    private $transaction_level = 0;
+
+    /**
+     * @var PDO $cacheHandler
+     */
+    private $cacheHandler;
     /**
      * @var array $tables                   an array containing all the initialized Doctrine_Table objects
      *                                      keys representing Doctrine_Table component names and values as Doctrine_Table objects
      */
-    private $tables   = array();
+    protected $tables         = array();
     /**
-     * @see Doctrine_Session::STATE_* constants
-     * @var boolean $state                  the current state of the session
+     * @var Doctrine_Validator $validator   transaction validator
      */
-    private $state       = 0;
+    protected $validator;
     /**
      * @var array $update                   two dimensional pending update list, the records in
      *                                      this list will be updated when transaction is committed
      */
-    private $update     = array(array());
+    protected $update       = array();
     /**
      * @var array $insert                   two dimensional pending insert list, the records in
      *                                      this list will be inserted when transaction is committed
      */
-    private $insert     = array(array());
+    protected $insert       = array();
     /**
      * @var array $delete                   two dimensional pending delete list, the records in
      *                                      this list will be deleted when transaction is committed
      */
-    private $delete     = array(array());
-    /** 
-     * @var integer $transaction_level      the nesting level of transactions, used by transaction methods
-     */
-    private $transaction_level = 0;
-    /**
-     * @var Doctrine_Validator $validator   transaction validator
-     */
-    private $validator;
-    /**
-     * @var PDO $cacheHandler
-     */
-    private $cacheHandler;
+    protected $delete       = array();
+
 
 
 
@@ -410,6 +412,9 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
      * @return void
      */
     public function bulkInsert() {
+        if(empty($this->insert))
+            return false;
+
         foreach($this->insert as $name => $inserts) {
             if( ! isset($inserts[0]))
                 continue;
@@ -431,7 +436,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
                 $stmt->closeCursor();
                 $increment = true;
             }
-            
+
 
             foreach($inserts as $k => $record) {
                 $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onPreSave($record);
@@ -443,6 +448,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
                     $id++;
                 }
 
+
                 $this->insert($record,$id);
                 // listen the onInsert event
                 $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onInsert($record);
@@ -450,7 +456,34 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
                 $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onSave($record);
             }
         }
-        $this->insert = array(array());
+        $this->insert = array();
+        return true;
+    }
+    /**
+     * returns maximum identifier values
+     *
+     * @param array $names          an array of component names
+     * @return array
+     */   
+    public function getMaximumValues(array $names) {
+        $values = array();
+        foreach($names as $name) {
+            $table     = $this->tables[$name];
+            $keys      = $table->getPrimaryKeys();
+            $tablename = $table->getTableName();
+
+            if(count($keys) == 1 && $keys[0] == "id") {
+                // record uses auto_increment column
+
+                $sql    = "SELECT MAX(id) FROM ".$tablename;
+                $stmt   = $this->dbh->query($sql);
+                $data   = $stmt->fetch(PDO::FETCH_NUM);
+                $values[$tablename] = $data[0];
+
+                $stmt->closeCursor();
+            }
+        }
+        return $values;
     }
     /**
      * bulkUpdate
@@ -477,7 +510,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
             if(isset($record))
                 $record->getTable()->getCache()->deleteMultiple($ids);
         }
-        $this->update = array(array());
+        $this->update = array();
     }
     /**
      * bulkDelete
@@ -499,7 +532,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
                 $record->getTable()->getCache()->deleteMultiple($ids);
             }
         }
-        $this->delete = array(array());
+        $this->delete = array();
     }
 
 
@@ -532,9 +565,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
      * @param Doctrine_Record $record
      * @return void
      */
-
     public function save(Doctrine_Record $record) {
-
         switch($record->getState()):
             case Doctrine_Record::STATE_TDIRTY:
                 $this->addInsert($record);
@@ -741,6 +772,25 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab
         $name = $record->getTable()->getComponentName();
         $this->delete[$name][] = $record;
     }
+    /**
+     * @return array
+     */
+    public function getInserts() {
+        return $this->insert;
+    }
+    /**
+     * @return array
+     */
+    public function getUpdates() {
+        return $this->update;
+    }
+    /**
+     * @return array
+     */
+    public function getDeletes() {
+        return $this->delete;
+    }
+
     /**
      * returns a string representation of this object
      * @return string

classes/Session/Mysql.class.php

@@ -34,5 +34,131 @@ class Doctrine_Session_Mysql extends Doctrine_Session_Common {
         return $ids;
     }
     */
+
+    /**
+     * returns maximum identifier values
+     *
+     * @param array $names          an array of component names
+     * @return array
+     */
+    public function getMaximumValues2(array $names) { 
+        $values = array();
+        foreach($names as $name) {
+            $table     = $this->tables[$name];
+            $keys      = $table->getPrimaryKeys();
+            $tablename = $table->getTableName();
+
+            if(count($keys) == 1 && $keys[0] == "id") {
+                // record uses auto_increment column
+
+                $sql[]    = "SELECT MAX(".$tablename.".id) as $tablename FROM ".$tablename;
+                $values[$tablename] = 0;
+                $array[] = $tablename;
+            }
+        }
+        $sql    = implode(" UNION ",$sql);
+        $stmt   = $this->getDBH()->query($sql);
+        $data   = $stmt->fetchAll(PDO::FETCH_NUM);
+
+        foreach($data as $k => $v) {
+            $values[$array[$k]] = $v[0];
+        }
+        return $values;
+    }
+    /**
+     * bulkInsert
+     * inserts all the objects in the pending insert list into database
+     *
+     * @return boolean
+     */
+    public function bulkInsert() {
+        if(empty($this->insert))
+            return false;
+
+        $values = $this->getMaximumValues(array_keys($this->insert));
+
+        foreach($this->insert as $name => $inserts) {
+            if( ! isset($inserts[0]))
+                continue;
+
+            $record    = $inserts[0];
+            $table     = $record->getTable();
+            $seq       = $table->getSequenceName();
+            $increment = false;
+            $id        = null;
+            $keys      = $table->getPrimaryKeys();
+            if(count($keys) == 1 && $keys[0] == "id") {
+
+                // record uses auto_increment column
+
+                $sql  = "SELECT MAX(id) FROM ".$record->getTable()->getTableName();
+                $stmt = $this->getDBH()->query($sql);
+                $data = $stmt->fetch(PDO::FETCH_NUM);
+                $id   = $data[0];
+                $stmt->closeCursor();
+                $increment = true;
+            }
+
+
+            $marks = array();
+            $params = array();
+            foreach($inserts as $k => $record) {
+                $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onPreSave($record);
+                // listen the onPreInsert event
+                $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onPreInsert($record);
+
+
+                if($increment) {
+                    // record uses auto_increment column
+                    $id++;
+                }
+
+
+                $array = $record->getModified();
+
+                foreach($record->getTable()->getInheritanceMap() as $k=>$v):
+                    $array[$k] = $v;
+                endforeach;
+
+                foreach($array as $k => $value) {
+                    if($value instanceof Doctrine_Record) {
+                        $array[$k] = $value->getID();
+                        $record->set($k,$value->getID());
+                    }
+                }
+                
+                if(isset($this->validator)) {
+                    if( ! $this->validator->validateRecord($record)) {
+                        continue;
+                    }
+                }
+
+                $key = implode(", ",array_keys($array));
+                if( ! isset($params[$key]))
+                    $params[$key] = array();
+
+                $marks[$key][] = "(".substr(str_repeat("?, ",count($array)),0,-2).")";
+                $params[$key] = array_merge($params[$key], array_values($array));
+
+                $record->setID($id);
+
+                // listen the onInsert event
+                $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onInsert($record);
+
+                $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onSave($record);
+            }
+
+            if( ! empty($marks)) {
+                foreach($marks as $key => $list) {
+                    $query = "INSERT INTO ".$table->getTableName()." (".$key.") VALUES ".implode(", ", $list);
+                    $stmt  = $this->getDBH()->prepare($query);
+                    $stmt->execute($params[$key]);
+                }
+            }
+        }
+        $this->insert = array();
+        return true;
+    }
+
 }
 ?>

tests/UnitTestCase.class.php

@@ -58,7 +58,6 @@ class Doctrine_UnitTestCase extends UnitTestCase {
 
         foreach($tables as $name) {
             $table = $this->session->getTable($name);
-
             $table->getCache()->deleteAll();
         }
 
@@ -73,14 +72,13 @@ class Doctrine_UnitTestCase extends UnitTestCase {
         $groups = new Doctrine_Collection($this->session->getTable("Group"));
 
         $groups[0]->name = "Drama Actors";
-        $groups[0]->save();
 
         $groups[1]->name = "Quality Actors";
-        $groups[1]->save();
+
 
         $groups[2]->name = "Action Actors";
         $groups[2]["Phonenumber"][0]->phonenumber = "123 123";
-        $groups[2]->save();
+        $groups->save();
 
         $users = new Doctrine_Collection($this->session->getTable("User"));
 

tests/run.php

@@ -23,8 +23,9 @@ error_reporting(E_ALL);
 
 $test = new GroupTest("Doctrine Framework Unit Tests");
 
-
 $test->addTestCase(new Doctrine_SessionTestCase());
+
+
 $test->addTestCase(new Doctrine_RecordTestCase());
 
 
@@ -42,7 +43,6 @@ $test->addTestCase(new Doctrine_EventListenerTestCase());
 
 $test->addTestCase(new Doctrine_BatchIteratorTestCase());
 
-
 $test->addTestCase(new Doctrine_DQL_ParserTestCase());
 
 $test->addTestCase(new Doctrine_ConfigurableTestCase());