From 83e4141639d026dfa93c4c2c0f9fe00a7cf56446 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 May 2017 15:17:38 +0000 Subject: [PATCH 1/3] Bugfix: dba::num_rows hadn't checked the object variable --- include/dba.php | 13 ++++++ include/dbclean.php | 97 +++++++++++++++++++++++++++++++++------------ 2 files changed, 85 insertions(+), 25 deletions(-) diff --git a/include/dba.php b/include/dba.php index 3af8522516..76ac538aa8 100644 --- a/include/dba.php +++ b/include/dba.php @@ -521,6 +521,16 @@ class dba { logger('Parameter mismatch. Query "'.$sql.'" - Parameters '.print_r($args, true), LOGGER_DEBUG); } + // beautify the SQL query - useful for "SHOW PROCESSLIST" + // This is safe because we bind the parameters later. + // The parameter values aren't part of the SQL. + $search = array("\n", "\r", " "); + $replace = array(' ', ' ', ' '); + do { + $oldsql = $sql; + $sql = str_replace($search, $replace, $sql); + } while ($oldsql != $sql); + $sql = self::$dbo->any_value_fallback($sql); if (x($a->config,'system') && x($a->config['system'], 'db_callstack')) { @@ -708,6 +718,9 @@ class dba { * @return int Number of rows */ static public function num_rows($stmt) { + if (!is_object($stmt)) { + return 0; + } switch (self::$dbo->driver) { case 'pdo': return $stmt->rowCount(); diff --git a/include/dbclean.php b/include/dbclean.php index 36f4f46e53..ff4993c4a0 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -19,7 +19,7 @@ function dbclean_run(&$argv, &$argc) { if ($stage == 0) { for ($i = 1; $i <= 7; $i++) { - if (!Config::get('system', 'finished-dbclean-'.$i)) { + if (!Config::get('system', 'finished-dbclean-'.$i, false)) { proc_run(PRIORITY_LOW, 'include/dbclean.php', $i); } } @@ -40,131 +40,178 @@ function remove_orphans($stage = 0) { $limit = 1000; if ($stage == 1) { - logger("Deleting old global item entries from item table without user copy"); - $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 - AND NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) - AND `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY LIMIT ".intval($limit)); + $last_id = Config::get('system', 'dbclean-last-id-1', 0); + + logger("Deleting old global item entries from item table without user copy. Last ID: ".$last_id); + $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 AND + NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) AND + `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY AND `id` >= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id); $count = dba::num_rows($r); if ($count > 0) { logger("found global item orphans: ".$count); while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; dba::delete('item', array('id' => $orphan["id"])); } } else { logger("No global item orphans found"); - } dba::close($r); - logger("Done deleting ".$count." old global item entries from item table without user copy"); + logger("Done deleting ".$count." old global item entries from item table without user copy. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-1', $last_id); // We will eventually set this value when we found a good way to delete these items in another way. // if ($count < $limit) { // Config::set('system', 'finished-dbclean-1', true); // } } elseif ($stage == 2) { - logger("Deleting items without parents"); - $r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit)); + $last_id = Config::get('system', 'dbclean-last-id-2', 0); + + logger("Deleting items without parents. Last ID: ".$last_id); + $r = dba::p("SELECT `id` FROM `item` + WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) + AND `id` >= ? ORDER BY `id` LIMIT ".intval($limit), $last_id); $count = dba::num_rows($r); if ($count > 0) { logger("found item orphans without parents: ".$count); while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; dba::delete('item', array('id' => $orphan["id"])); } } else { logger("No item orphans without parents found"); } dba::close($r); - logger("Done deleting ".$count." items without parents"); + logger("Done deleting ".$count." items without parents. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-2', $last_id); if ($count < $limit) { Config::set('system', 'finished-dbclean-2', true); } } elseif ($stage == 3) { - logger("Deleting orphaned data from thread table"); - $r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit)); + $last_id = Config::get('system', 'dbclean-last-id-3', 0); + + logger("Deleting orphaned data from thread table. Last ID: ".$last_id); + $r = dba::p("SELECT `iid` FROM `thread` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) AND `iid` >= ? + ORDER BY `iid` LIMIT ".intval($limit), $last_id); $count = dba::num_rows($r); if ($count > 0) { logger("found thread orphans: ".$count); while ($orphan = dba::fetch($r)) { + $last_id = $orphan["iid"]; dba::delete('thread', array('iid' => $orphan["iid"])); } } else { logger("No thread orphans found"); } dba::close($r); - logger("Done deleting ".$count." orphaned data from thread table"); + logger("Done deleting ".$count." orphaned data from thread table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-3', $last_id); if ($count < $limit) { Config::set('system', 'finished-dbclean-3', true); } } elseif ($stage == 4) { - logger("Deleting orphaned data from notify table"); - $r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit)); + $last_id = Config::get('system', 'dbclean-last-id-4', 0); + + logger("Deleting orphaned data from notify table. Last ID: ".$last_id); + $r = dba::p("SELECT `iid`, `id` FROM `notify` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) AND `id` >= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id); $count = dba::num_rows($r); if ($count > 0) { logger("found notify orphans: ".$count); while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; dba::delete('notify', array('iid' => $orphan["iid"])); } } else { logger("No notify orphans found"); } dba::close($r); - logger("Done deleting ".$count." orphaned data from notify table"); + logger("Done deleting ".$count." orphaned data from notify table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-4', $last_id); if ($count < $limit) { Config::set('system', 'finished-dbclean-4', true); } } elseif ($stage == 5) { - logger("Deleting orphaned data from notify-threads table"); - $r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit)); + $last_id = Config::get('system', 'dbclean-last-id-5', 0); + + logger("Deleting orphaned data from notify-threads table. Last ID: ".$last_id); + $r = dba::p("SELECT `id` FROM `notify-threads` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) AND `id` >= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id); $count = dba::num_rows($r); if ($count > 0) { logger("found notify-threads orphans: ".$count); while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; dba::delete('notify-threads', array('id' => $orphan["id"])); } } else { logger("No notify-threads orphans found"); } dba::close($r); - logger("Done deleting ".$count." orphaned data from notify-threads table"); + logger("Done deleting ".$count." orphaned data from notify-threads table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-5', $last_id); if ($count < $limit) { Config::set('system', 'finished-dbclean-5', true); } } elseif ($stage == 6) { - logger("Deleting orphaned data from sign table"); - $r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit)); + $last_id = Config::get('system', 'dbclean-last-id-6', 0); + + logger("Deleting orphaned data from sign table. Last ID: ".$last_id); + $r = dba::p("SELECT `iid`, `id` FROM `sign` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) AND `id` >= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id); $count = dba::num_rows($r); if ($count > 0) { logger("found sign orphans: ".$count); while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; dba::delete('sign', array('iid' => $orphan["iid"])); } } else { logger("No sign orphans found"); } dba::close($r); - logger("Done deleting ".$count." orphaned data from sign table"); + logger("Done deleting ".$count." orphaned data from sign table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-6', $last_id); if ($count < $limit) { Config::set('system', 'finished-dbclean-6', true); } } elseif ($stage == 7) { - logger("Deleting orphaned data from term table"); - $r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit)); + $last_id = Config::get('system', 'dbclean-last-id-7', 0); + + logger("Deleting orphaned data from term table. Last ID: ".$last_id); + $r = dba::p("SELECT `oid`, `tid` FROM `term` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) AND `tid` >= ? + ORDER BY `tid` LIMIT ".intval($limit), $last_id); $count = dba::num_rows($r); if ($count > 0) { logger("found term orphans: ".$count); while ($orphan = dba::fetch($r)) { + $last_id = $orphan["tid"]; dba::delete('term', array('oid' => $orphan["oid"])); } } else { logger("No term orphans found"); } dba::close($r); - logger("Done deleting ".$count." orphaned data from term table"); + logger("Done deleting ".$count." orphaned data from term table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-7', $last_id); if ($count < $limit) { Config::set('system', 'finished-dbclean-7', true); From 124690cc4d2a00a350b6b2b1c592e0fb0134e070 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 May 2017 21:06:17 +0000 Subject: [PATCH 2/3] dba: Beautification is now a separate function --- include/dba.php | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/include/dba.php b/include/dba.php index 76ac538aa8..ccce9ed816 100644 --- a/include/dba.php +++ b/include/dba.php @@ -459,6 +459,27 @@ class dba { return $sql; } + /** + * @brief beautifies the query - seful for "SHOW PROCESSLIST" + * + * This is safe when we bind the parameters later. + * The parameter values aren't part of the SQL. + * + * @param string $sql An SQL string without the values + * @return string The input SQL string modified if necessary. + */ + public function clean_query($sql) { + $search = array("\t", "\n", "\r", " "); + $replace = array(' ', ' ', ' ', ' '); + do { + $oldsql = $sql; + $sql = str_replace($search, $replace, $sql); + } while ($oldsql != $sql); + + return $sql; + } + + /** * @brief Replaces the ? placeholders with the parameters in the $args array * @@ -521,16 +542,7 @@ class dba { logger('Parameter mismatch. Query "'.$sql.'" - Parameters '.print_r($args, true), LOGGER_DEBUG); } - // beautify the SQL query - useful for "SHOW PROCESSLIST" - // This is safe because we bind the parameters later. - // The parameter values aren't part of the SQL. - $search = array("\n", "\r", " "); - $replace = array(' ', ' ', ' '); - do { - $oldsql = $sql; - $sql = str_replace($search, $replace, $sql); - } while ($oldsql != $sql); - + $sql = self::$dbo->clean_query($sql); $sql = self::$dbo->any_value_fallback($sql); if (x($a->config,'system') && x($a->config['system'], 'db_callstack')) { @@ -1213,6 +1225,7 @@ function q($sql) { unset($args[0]); if ($db && $db->connected) { + $sql = $db->clean_query($sql); $sql = $db->any_value_fallback($sql); $stmt = @vsprintf($sql,$args); // Disabled warnings //logger("dba: q: $stmt", LOGGER_ALL); @@ -1250,6 +1263,7 @@ function qu($sql) { unset($args[0]); if ($db && $db->connected) { + $sql = $db->clean_query($sql); $sql = $db->any_value_fallback($sql); $stmt = @vsprintf($sql,$args); // Disabled warnings if ($stmt === false) From 41f94a62fcc14022f7d2401a3adefeaa111d0628 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 16 May 2017 06:00:01 +0000 Subject: [PATCH 3/3] Spelling --- include/dba.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dba.php b/include/dba.php index ccce9ed816..b9e6c32d56 100644 --- a/include/dba.php +++ b/include/dba.php @@ -460,7 +460,7 @@ class dba { } /** - * @brief beautifies the query - seful for "SHOW PROCESSLIST" + * @brief beautifies the query - useful for "SHOW PROCESSLIST" * * This is safe when we bind the parameters later. * The parameter values aren't part of the SQL.