From c435286eb0b191296110becad416a0fa58e6c5c4 Mon Sep 17 00:00:00 2001 From: doctrine Date: Sat, 22 Apr 2006 09:08:02 +0000 Subject: [PATCH] Mysql bulk insert support => huge performance increase for mysql session --- classes/DQL/Parser.class.php | 2 +- classes/Session.class.php | 112 ++++++++++++++++++++-------- classes/Session/Mysql.class.php | 126 ++++++++++++++++++++++++++++++++ tests/UnitTestCase.class.php | 6 +- tests/run.php | 4 +- 5 files changed, 212 insertions(+), 38 deletions(-) diff --git a/classes/DQL/Parser.class.php b/classes/DQL/Parser.class.php index 72ec588bf..7b601bcb6 100644 --- a/classes/DQL/Parser.class.php +++ b/classes/DQL/Parser.class.php @@ -725,7 +725,7 @@ class Doctrine_DQL_Parser { $reference = implode(".",$a); $objTable = $this->session->getTable(end($a)); $where = $objTable->getTableName().".".$field." ".$operator." ".$value; - if(count($a) > 1 AND isset($a[1])) { + if(count($a) > 1 && isset($a[1])) { $root = $a[0]; $fk = $this->tnames[$root]->getForeignKey($a[1]); if($fk instanceof Doctrine_Association) { diff --git a/classes/Session.class.php b/classes/Session.class.php index 8b28fe068..4d8e47f0a 100644 --- a/classes/Session.class.php +++ b/classes/Session.class.php @@ -29,43 +29,45 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab * @var $dbh the database handle */ private $dbh; - /** - * @var array $tables an array containing all the initialized Doctrine_Table objects - * keys representing Doctrine_Table component names and values as Doctrine_Table objects - */ - private $tables = array(); /** * @see Doctrine_Session::STATE_* constants * @var boolean $state the current state of the session */ - private $state = 0; + private $state = 0; /** - * @var array $update two dimensional pending update list, the records in - * this list will be updated when transaction is committed - */ - private $update = array(array()); - /** - * @var array $insert two dimensional pending insert list, the records in - * this list will be inserted when transaction is committed - */ - private $insert = array(array()); - /** - * @var array $delete two dimensional pending delete list, the records in - * this list will be deleted when transaction is committed - */ - private $delete = array(array()); - /** * @var integer $transaction_level the nesting level of transactions, used by transaction methods */ private $transaction_level = 0; - /** - * @var Doctrine_Validator $validator transaction validator - */ - private $validator; + /** * @var PDO $cacheHandler */ private $cacheHandler; + /** + * @var array $tables an array containing all the initialized Doctrine_Table objects + * keys representing Doctrine_Table component names and values as Doctrine_Table objects + */ + protected $tables = array(); + /** + * @var Doctrine_Validator $validator transaction validator + */ + protected $validator; + /** + * @var array $update two dimensional pending update list, the records in + * this list will be updated when transaction is committed + */ + protected $update = array(); + /** + * @var array $insert two dimensional pending insert list, the records in + * this list will be inserted when transaction is committed + */ + protected $insert = array(); + /** + * @var array $delete two dimensional pending delete list, the records in + * this list will be deleted when transaction is committed + */ + protected $delete = array(); + @@ -410,6 +412,9 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab * @return void */ public function bulkInsert() { + if(empty($this->insert)) + return false; + foreach($this->insert as $name => $inserts) { if( ! isset($inserts[0])) continue; @@ -431,7 +436,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab $stmt->closeCursor(); $increment = true; } - + foreach($inserts as $k => $record) { $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onPreSave($record); @@ -443,6 +448,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab $id++; } + $this->insert($record,$id); // listen the onInsert event $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onInsert($record); @@ -450,7 +456,34 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onSave($record); } } - $this->insert = array(array()); + $this->insert = array(); + return true; + } + /** + * returns maximum identifier values + * + * @param array $names an array of component names + * @return array + */ + public function getMaximumValues(array $names) { + $values = array(); + foreach($names as $name) { + $table = $this->tables[$name]; + $keys = $table->getPrimaryKeys(); + $tablename = $table->getTableName(); + + if(count($keys) == 1 && $keys[0] == "id") { + // record uses auto_increment column + + $sql = "SELECT MAX(id) FROM ".$tablename; + $stmt = $this->dbh->query($sql); + $data = $stmt->fetch(PDO::FETCH_NUM); + $values[$tablename] = $data[0]; + + $stmt->closeCursor(); + } + } + return $values; } /** * bulkUpdate @@ -477,7 +510,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab if(isset($record)) $record->getTable()->getCache()->deleteMultiple($ids); } - $this->update = array(array()); + $this->update = array(); } /** * bulkDelete @@ -499,7 +532,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab $record->getTable()->getCache()->deleteMultiple($ids); } } - $this->delete = array(array()); + $this->delete = array(); } @@ -532,9 +565,7 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab * @param Doctrine_Record $record * @return void */ - public function save(Doctrine_Record $record) { - switch($record->getState()): case Doctrine_Record::STATE_TDIRTY: $this->addInsert($record); @@ -741,6 +772,25 @@ abstract class Doctrine_Session extends Doctrine_Configurable implements Countab $name = $record->getTable()->getComponentName(); $this->delete[$name][] = $record; } + /** + * @return array + */ + public function getInserts() { + return $this->insert; + } + /** + * @return array + */ + public function getUpdates() { + return $this->update; + } + /** + * @return array + */ + public function getDeletes() { + return $this->delete; + } + /** * returns a string representation of this object * @return string diff --git a/classes/Session/Mysql.class.php b/classes/Session/Mysql.class.php index bb00bd03a..a2e27d910 100644 --- a/classes/Session/Mysql.class.php +++ b/classes/Session/Mysql.class.php @@ -34,5 +34,131 @@ class Doctrine_Session_Mysql extends Doctrine_Session_Common { return $ids; } */ + + /** + * returns maximum identifier values + * + * @param array $names an array of component names + * @return array + */ + public function getMaximumValues2(array $names) { + $values = array(); + foreach($names as $name) { + $table = $this->tables[$name]; + $keys = $table->getPrimaryKeys(); + $tablename = $table->getTableName(); + + if(count($keys) == 1 && $keys[0] == "id") { + // record uses auto_increment column + + $sql[] = "SELECT MAX(".$tablename.".id) as $tablename FROM ".$tablename; + $values[$tablename] = 0; + $array[] = $tablename; + } + } + $sql = implode(" UNION ",$sql); + $stmt = $this->getDBH()->query($sql); + $data = $stmt->fetchAll(PDO::FETCH_NUM); + + foreach($data as $k => $v) { + $values[$array[$k]] = $v[0]; + } + return $values; + } + /** + * bulkInsert + * inserts all the objects in the pending insert list into database + * + * @return boolean + */ + public function bulkInsert() { + if(empty($this->insert)) + return false; + + $values = $this->getMaximumValues(array_keys($this->insert)); + + foreach($this->insert as $name => $inserts) { + if( ! isset($inserts[0])) + continue; + + $record = $inserts[0]; + $table = $record->getTable(); + $seq = $table->getSequenceName(); + $increment = false; + $id = null; + $keys = $table->getPrimaryKeys(); + if(count($keys) == 1 && $keys[0] == "id") { + + // record uses auto_increment column + + $sql = "SELECT MAX(id) FROM ".$record->getTable()->getTableName(); + $stmt = $this->getDBH()->query($sql); + $data = $stmt->fetch(PDO::FETCH_NUM); + $id = $data[0]; + $stmt->closeCursor(); + $increment = true; + } + + + $marks = array(); + $params = array(); + foreach($inserts as $k => $record) { + $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onPreSave($record); + // listen the onPreInsert event + $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onPreInsert($record); + + + if($increment) { + // record uses auto_increment column + $id++; + } + + + $array = $record->getModified(); + + foreach($record->getTable()->getInheritanceMap() as $k=>$v): + $array[$k] = $v; + endforeach; + + foreach($array as $k => $value) { + if($value instanceof Doctrine_Record) { + $array[$k] = $value->getID(); + $record->set($k,$value->getID()); + } + } + + if(isset($this->validator)) { + if( ! $this->validator->validateRecord($record)) { + continue; + } + } + + $key = implode(", ",array_keys($array)); + if( ! isset($params[$key])) + $params[$key] = array(); + + $marks[$key][] = "(".substr(str_repeat("?, ",count($array)),0,-2).")"; + $params[$key] = array_merge($params[$key], array_values($array)); + + $record->setID($id); + + // listen the onInsert event + $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onInsert($record); + + $record->getTable()->getAttribute(Doctrine::ATTR_LISTENER)->onSave($record); + } + + if( ! empty($marks)) { + foreach($marks as $key => $list) { + $query = "INSERT INTO ".$table->getTableName()." (".$key.") VALUES ".implode(", ", $list); + $stmt = $this->getDBH()->prepare($query); + $stmt->execute($params[$key]); + } + } + } + $this->insert = array(); + return true; + } + } ?> diff --git a/tests/UnitTestCase.class.php b/tests/UnitTestCase.class.php index 465581dfb..e5565c0d8 100644 --- a/tests/UnitTestCase.class.php +++ b/tests/UnitTestCase.class.php @@ -58,7 +58,6 @@ class Doctrine_UnitTestCase extends UnitTestCase { foreach($tables as $name) { $table = $this->session->getTable($name); - $table->getCache()->deleteAll(); } @@ -73,14 +72,13 @@ class Doctrine_UnitTestCase extends UnitTestCase { $groups = new Doctrine_Collection($this->session->getTable("Group")); $groups[0]->name = "Drama Actors"; - $groups[0]->save(); $groups[1]->name = "Quality Actors"; - $groups[1]->save(); + $groups[2]->name = "Action Actors"; $groups[2]["Phonenumber"][0]->phonenumber = "123 123"; - $groups[2]->save(); + $groups->save(); $users = new Doctrine_Collection($this->session->getTable("User")); diff --git a/tests/run.php b/tests/run.php index 32a1afc63..d4df74521 100644 --- a/tests/run.php +++ b/tests/run.php @@ -23,8 +23,9 @@ error_reporting(E_ALL); $test = new GroupTest("Doctrine Framework Unit Tests"); - $test->addTestCase(new Doctrine_SessionTestCase()); + + $test->addTestCase(new Doctrine_RecordTestCase()); @@ -42,7 +43,6 @@ $test->addTestCase(new Doctrine_EventListenerTestCase()); $test->addTestCase(new Doctrine_BatchIteratorTestCase()); - $test->addTestCase(new Doctrine_DQL_ParserTestCase()); $test->addTestCase(new Doctrine_ConfigurableTestCase());