From 1c7fd310d21099e726438e5a9809bca56ebe94cf Mon Sep 17 00:00:00 2001 From: Adam Magness Date: Thu, 25 Jan 2018 10:08:56 -0500 Subject: [PATCH 1/3] Create PostUpdate class create new class and update call from worker --- src/Database/PostUpdate.php | 276 ++++++++++++++++++++++++++++++++++++ src/Worker/CronJobs.php | 9 +- 2 files changed, 279 insertions(+), 6 deletions(-) create mode 100644 src/Database/PostUpdate.php diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php new file mode 100644 index 000000000..f43985ac3 --- /dev/null +++ b/src/Database/PostUpdate.php @@ -0,0 +1,276 @@ += 1192) + return true; + + // Check if the first step is done (Setting "gcontact-id" in the item table) + $r = q("SELECT `author-link`, `author-name`, `author-avatar`, `uid`, `network` FROM `item` WHERE `gcontact-id` = 0 LIMIT 1000"); + if (!$r) { + // Are there unfinished entries in the thread table? + $r = q("SELECT COUNT(*) AS `total` FROM `thread` + INNER JOIN `item` ON `item`.`id` =`thread`.`iid` + WHERE `thread`.`gcontact-id` = 0 AND + (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); + + if ($r && ($r[0]["total"] == 0)) { + Config::set("system", "post_update_version", 1192); + return true; + } + + // Update the thread table from the item table + q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid` + SET `thread`.`gcontact-id` = `item`.`gcontact-id` + WHERE `thread`.`gcontact-id` = 0 AND + (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); + + return false; + } + + $item_arr = []; + foreach ($r as $item) { + $index = $item["author-link"]."-".$item["uid"]; + $item_arr[$index] = ["author-link" => $item["author-link"], + "uid" => $item["uid"], + "network" => $item["network"]]; + } + + // Set the "gcontact-id" in the item table and add a new gcontact entry if needed + foreach ($item_arr as $item) { + $gcontact_id = GContact::getId(["url" => $item['author-link'], "network" => $item['network'], + "photo" => $item['author-avatar'], "name" => $item['author-name']]); + q("UPDATE `item` SET `gcontact-id` = %d WHERE `uid` = %d AND `author-link` = '%s' AND `gcontact-id` = 0", + intval($gcontact_id), intval($item["uid"]), dbesc($item["author-link"])); + } + return false; + } + + /** + * @brief Updates the "global" field in the item table + * + * @return bool "true" when the job is done + */ + private static function update1194() + { + // Was the script completed? + if (Config::get("system", "post_update_version") >= 1194) + return true; + + logger("Start", LOGGER_DEBUG); + + $end_id = Config::get("system", "post_update_1194_end"); + if (!$end_id) { + $r = q("SELECT `id` FROM `item` WHERE `uid` != 0 ORDER BY `id` DESC LIMIT 1"); + if ($r) { + Config::set("system", "post_update_1194_end", $r[0]["id"]); + $end_id = Config::get("system", "post_update_1194_end"); + } + } + + logger("End ID: ".$end_id, LOGGER_DEBUG); + + $start_id = Config::get("system", "post_update_1194_start"); + + $query1 = "SELECT `item`.`id` FROM `item` "; + + $query2 = "INNER JOIN `item` AS `shadow` ON `item`.`uri` = `shadow`.`uri` AND `shadow`.`uid` = 0 "; + + $query3 = "WHERE `item`.`uid` != 0 AND `item`.`id` >= %d AND `item`.`id` <= %d + AND `item`.`visible` AND NOT `item`.`private` + AND NOT `item`.`deleted` AND NOT `item`.`moderated` + AND `item`.`network` IN ('%s', '%s', '%s', '') + AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' + AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' + AND NOT `item`.`global`"; + + $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 1", + intval($start_id), intval($end_id), + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); + if (!$r) { + Config::set("system", "post_update_version", 1194); + logger("Update is done", LOGGER_DEBUG); + return true; + } else { + Config::set("system", "post_update_1194_start", $r[0]["id"]); + $start_id = Config::get("system", "post_update_1194_start"); + } + + logger("Start ID: ".$start_id, LOGGER_DEBUG); + + $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 1000,1", + intval($start_id), intval($end_id), + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); + if ($r) + $pos_id = $r[0]["id"]; + else + $pos_id = $end_id; + + logger("Progress: Start: ".$start_id." position: ".$pos_id." end: ".$end_id, LOGGER_DEBUG); + + q("UPDATE `item` ".$query2." SET `item`.`global` = 1 ".$query3, + intval($start_id), intval($pos_id), + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); + + logger("Done", LOGGER_DEBUG); + } + + /** + * @brief set the author-id and owner-id in all item entries + * + * This job has to be started multiple times until all entries are set. + * It isn't started in the update function since it would consume too much time and can be done in the background. + * + * @return bool "true" when the job is done + */ + private static function update1198() + { + // Was the script completed? + if (Config::get("system", "post_update_version") >= 1198) + return true; + + logger("Start", LOGGER_DEBUG); + + // Check if the first step is done (Setting "author-id" and "owner-id" in the item table) + $r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 100"); + if (!$r) { + // Are there unfinished entries in the thread table? + $r = q("SELECT COUNT(*) AS `total` FROM `thread` + INNER JOIN `item` ON `item`.`id` =`thread`.`iid` + WHERE `thread`.`author-id` = 0 AND `thread`.`owner-id` = 0 AND + (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); + + if ($r && ($r[0]["total"] == 0)) { + Config::set("system", "post_update_version", 1198); + logger("Done", LOGGER_DEBUG); + return true; + } + + // Update the thread table from the item table + $r = q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid` + SET `thread`.`author-id` = `item`.`author-id`, + `thread`.`owner-id` = `item`.`owner-id` + WHERE `thread`.`author-id` = 0 AND `thread`.`owner-id` = 0 AND + (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); + + logger("Updated threads", LOGGER_DEBUG); + if (DBM::is_result($r)) { + Config::set("system", "post_update_version", 1198); + logger("Done", LOGGER_DEBUG); + return true; + } + return false; + } + + logger("Query done", LOGGER_DEBUG); + + $item_arr = []; + foreach ($r AS $item) { + $index = $item["author-link"]."-".$item["owner-link"]."-".$item["uid"]; + $item_arr[$index] = ["author-link" => $item["author-link"], + "owner-link" => $item["owner-link"], + "uid" => $item["uid"]]; + } + + // Set the "gcontact-id" in the item table and add a new gcontact entry if needed + foreach ($item_arr AS $item) { + $author_id = Contact::getIdForURL($item["author-link"], 0); + $owner_id = Contact::getIdForURL($item["owner-link"], 0); + + if ($author_id == 0) + $author_id = -1; + + if ($owner_id == 0) + $owner_id = -1; + + q("UPDATE `item` SET `author-id` = %d, `owner-id` = %d + WHERE `uid` = %d AND `author-link` = '%s' AND `owner-link` = '%s' + AND `author-id` = 0 AND `owner-id` = 0", + intval($author_id), intval($owner_id), intval($item["uid"]), + dbesc($item["author-link"]), dbesc($item["owner-link"])); + } + + logger("Updated items", LOGGER_DEBUG); + return false; + } + + /** + * @brief update the "last-item" field in the "self" contact + * + * This field avoids cost intensive calls in the admin panel and in "nodeinfo" + * + * @return bool "true" when the job is done + */ + private static function update1206() + { + // Was the script completed? + if (Config::get("system", "post_update_version") >= 1206) + return true; + + logger("Start", LOGGER_DEBUG); + $r = q("SELECT `contact`.`id`, `contact`.`last-item`, + (SELECT MAX(`changed`) FROM `item` USE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` + FROM `user` + INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`"); + + if (!DBM::is_result($r)) { + return false; + } + foreach ($r AS $user) { + if (!empty($user["lastitem_date"]) && ($user["lastitem_date"] > $user["last-item"])) { + q("UPDATE `contact` SET `last-item` = '%s' WHERE `id` = %d", + dbesc($user["lastitem_date"]), + intval($user["id"])); + } + } + + Config::set("system", "post_update_version", 1206); + logger("Done", LOGGER_DEBUG); + return true; + } +} diff --git a/src/Worker/CronJobs.php b/src/Worker/CronJobs.php index 9e130d9ef..25113dada 100644 --- a/src/Worker/CronJobs.php +++ b/src/Worker/CronJobs.php @@ -1,15 +1,14 @@ Date: Thu, 25 Jan 2018 10:51:10 -0500 Subject: [PATCH 2/3] Update functions and dba update some functions and add dba where possible --- src/Database/PostUpdate.php | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index f43985ac3..dba2b3212 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -47,11 +47,12 @@ class PostUpdate private static function update1192() { // Was the script completed? - if (Config::get("system", "post_update_version") >= 1192) + if (Config::get("system", "post_update_version") >= 1192) { return true; + } // Check if the first step is done (Setting "gcontact-id" in the item table) - $r = q("SELECT `author-link`, `author-name`, `author-avatar`, `uid`, `network` FROM `item` WHERE `gcontact-id` = 0 LIMIT 1000"); + $r = dba::select('item', ['author-link', 'author-name', 'author-avatar', 'uid', 'network'], ['gcontact-id' => 0], ['limit' => 1000]); if (!$r) { // Are there unfinished entries in the thread table? $r = q("SELECT COUNT(*) AS `total` FROM `thread` @@ -85,8 +86,7 @@ class PostUpdate foreach ($item_arr as $item) { $gcontact_id = GContact::getId(["url" => $item['author-link'], "network" => $item['network'], "photo" => $item['author-avatar'], "name" => $item['author-name']]); - q("UPDATE `item` SET `gcontact-id` = %d WHERE `uid` = %d AND `author-link` = '%s' AND `gcontact-id` = 0", - intval($gcontact_id), intval($item["uid"]), dbesc($item["author-link"])); + dba::update('item', ['gcontact-id' => $gcontact_id], ['uid' => $item['uid'], 'author-link' => $item['author-link'], 'gcontact-id' => 0]); } return false; } @@ -99,8 +99,9 @@ class PostUpdate private static function update1194() { // Was the script completed? - if (Config::get("system", "post_update_version") >= 1194) + if (Config::get("system", "post_update_version") >= 1194) { return true; + } logger("Start", LOGGER_DEBUG); @@ -171,13 +172,14 @@ class PostUpdate private static function update1198() { // Was the script completed? - if (Config::get("system", "post_update_version") >= 1198) + if (Config::get("system", "post_update_version") >= 1198) { return true; + } logger("Start", LOGGER_DEBUG); // Check if the first step is done (Setting "author-id" and "owner-id" in the item table) - $r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 100"); + $r = dba::select('item', ['author-link', 'owner-link', 'uid'], ['author-id' => 0, 'owner-id' => 0], ['limit' => 1000]); if (!$r) { // Are there unfinished entries in the thread table? $r = q("SELECT COUNT(*) AS `total` FROM `thread` @@ -210,7 +212,7 @@ class PostUpdate logger("Query done", LOGGER_DEBUG); $item_arr = []; - foreach ($r AS $item) { + foreach ($r as $item) { $index = $item["author-link"]."-".$item["owner-link"]."-".$item["uid"]; $item_arr[$index] = ["author-link" => $item["author-link"], "owner-link" => $item["owner-link"], @@ -218,7 +220,7 @@ class PostUpdate } // Set the "gcontact-id" in the item table and add a new gcontact entry if needed - foreach ($item_arr AS $item) { + foreach ($item_arr as $item) { $author_id = Contact::getIdForURL($item["author-link"], 0); $owner_id = Contact::getIdForURL($item["owner-link"], 0); @@ -228,11 +230,7 @@ class PostUpdate if ($owner_id == 0) $owner_id = -1; - q("UPDATE `item` SET `author-id` = %d, `owner-id` = %d - WHERE `uid` = %d AND `author-link` = '%s' AND `owner-link` = '%s' - AND `author-id` = 0 AND `owner-id` = 0", - intval($author_id), intval($owner_id), intval($item["uid"]), - dbesc($item["author-link"]), dbesc($item["owner-link"])); + dba::update('item', ['author-id' => $author_id, 'owner-id' => $owner_id], ['uid' => $item['uid'], 'author-link' => $item['author-link'], 'owner-link' => $item['owner-link'], 'author-id' => 0, 'owner-id' => 0]); } logger("Updated items", LOGGER_DEBUG); @@ -249,8 +247,9 @@ class PostUpdate private static function update1206() { // Was the script completed? - if (Config::get("system", "post_update_version") >= 1206) + if (Config::get("system", "post_update_version") >= 1206) { return true; + } logger("Start", LOGGER_DEBUG); $r = q("SELECT `contact`.`id`, `contact`.`last-item`, @@ -261,11 +260,9 @@ class PostUpdate if (!DBM::is_result($r)) { return false; } - foreach ($r AS $user) { + foreach ($r as $user) { if (!empty($user["lastitem_date"]) && ($user["lastitem_date"] > $user["last-item"])) { - q("UPDATE `contact` SET `last-item` = '%s' WHERE `id` = %d", - dbesc($user["lastitem_date"]), - intval($user["id"])); + dba::update('contact', ['last-item' => $user['lastitem_date']], ['id' => $user['id']]); } } From 5951ceeb6f0aa027c26ff1f1d12cfd7b45c03930 Mon Sep 17 00:00:00 2001 From: Adam Magness Date: Thu, 25 Jan 2018 10:52:00 -0500 Subject: [PATCH 3/3] Remove old file remove include/post_update.php --- include/post_update.php | 265 ---------------------------------------- 1 file changed, 265 deletions(-) delete mode 100644 include/post_update.php diff --git a/include/post_update.php b/include/post_update.php deleted file mode 100644 index 44e0a717c..000000000 --- a/include/post_update.php +++ /dev/null @@ -1,265 +0,0 @@ -= 1192) - return true; - - // Check if the first step is done (Setting "gcontact-id" in the item table) - $r = q("SELECT `author-link`, `author-name`, `author-avatar`, `uid`, `network` FROM `item` WHERE `gcontact-id` = 0 LIMIT 1000"); - if (!$r) { - // Are there unfinished entries in the thread table? - $r = q("SELECT COUNT(*) AS `total` FROM `thread` - INNER JOIN `item` ON `item`.`id` =`thread`.`iid` - WHERE `thread`.`gcontact-id` = 0 AND - (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); - - if ($r && ($r[0]["total"] == 0)) { - Config::set("system", "post_update_version", 1192); - return true; - } - - // Update the thread table from the item table - q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid` - SET `thread`.`gcontact-id` = `item`.`gcontact-id` - WHERE `thread`.`gcontact-id` = 0 AND - (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); - - return false; - } - - $item_arr = []; - foreach ($r AS $item) { - $index = $item["author-link"]."-".$item["uid"]; - $item_arr[$index] = ["author-link" => $item["author-link"], - "uid" => $item["uid"], - "network" => $item["network"]]; - } - - // Set the "gcontact-id" in the item table and add a new gcontact entry if needed - foreach ($item_arr AS $item) { - $gcontact_id = GContact::getId(["url" => $item['author-link'], "network" => $item['network'], - "photo" => $item['author-avatar'], "name" => $item['author-name']]); - q("UPDATE `item` SET `gcontact-id` = %d WHERE `uid` = %d AND `author-link` = '%s' AND `gcontact-id` = 0", - intval($gcontact_id), intval($item["uid"]), dbesc($item["author-link"])); - } - return false; -} - -/** - * @brief Updates the "global" field in the item table - * - * @return bool "true" when the job is done - */ -function post_update_1194() { - - // Was the script completed? - if (Config::get("system", "post_update_version") >= 1194) - return true; - - logger("Start", LOGGER_DEBUG); - - $end_id = Config::get("system", "post_update_1194_end"); - if (!$end_id) { - $r = q("SELECT `id` FROM `item` WHERE `uid` != 0 ORDER BY `id` DESC LIMIT 1"); - if ($r) { - Config::set("system", "post_update_1194_end", $r[0]["id"]); - $end_id = Config::get("system", "post_update_1194_end"); - } - } - - logger("End ID: ".$end_id, LOGGER_DEBUG); - - $start_id = Config::get("system", "post_update_1194_start"); - - $query1 = "SELECT `item`.`id` FROM `item` "; - - $query2 = "INNER JOIN `item` AS `shadow` ON `item`.`uri` = `shadow`.`uri` AND `shadow`.`uid` = 0 "; - - $query3 = "WHERE `item`.`uid` != 0 AND `item`.`id` >= %d AND `item`.`id` <= %d - AND `item`.`visible` AND NOT `item`.`private` - AND NOT `item`.`deleted` AND NOT `item`.`moderated` - AND `item`.`network` IN ('%s', '%s', '%s', '') - AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' - AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' - AND NOT `item`.`global`"; - - $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 1", - intval($start_id), intval($end_id), - dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); - if (!$r) { - Config::set("system", "post_update_version", 1194); - logger("Update is done", LOGGER_DEBUG); - return true; - } else { - Config::set("system", "post_update_1194_start", $r[0]["id"]); - $start_id = Config::get("system", "post_update_1194_start"); - } - - logger("Start ID: ".$start_id, LOGGER_DEBUG); - - $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 1000,1", - intval($start_id), intval($end_id), - dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); - if ($r) - $pos_id = $r[0]["id"]; - else - $pos_id = $end_id; - - logger("Progress: Start: ".$start_id." position: ".$pos_id." end: ".$end_id, LOGGER_DEBUG); - - q("UPDATE `item` ".$query2." SET `item`.`global` = 1 ".$query3, - intval($start_id), intval($pos_id), - dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); - - logger("Done", LOGGER_DEBUG); -} - -/** - * @brief set the author-id and owner-id in all item entries - * - * This job has to be started multiple times until all entries are set. - * It isn't started in the update function since it would consume too much time and can be done in the background. - * - * @return bool "true" when the job is done - */ -function post_update_1198() { - - // Was the script completed? - if (Config::get("system", "post_update_version") >= 1198) - return true; - - logger("Start", LOGGER_DEBUG); - - // Check if the first step is done (Setting "author-id" and "owner-id" in the item table) - $r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 100"); - if (!$r) { - // Are there unfinished entries in the thread table? - $r = q("SELECT COUNT(*) AS `total` FROM `thread` - INNER JOIN `item` ON `item`.`id` =`thread`.`iid` - WHERE `thread`.`author-id` = 0 AND `thread`.`owner-id` = 0 AND - (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); - - if ($r && ($r[0]["total"] == 0)) { - Config::set("system", "post_update_version", 1198); - logger("Done", LOGGER_DEBUG); - return true; - } - - // Update the thread table from the item table - $r = q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid` - SET `thread`.`author-id` = `item`.`author-id`, - `thread`.`owner-id` = `item`.`owner-id` - WHERE `thread`.`author-id` = 0 AND `thread`.`owner-id` = 0 AND - (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); - - logger("Updated threads", LOGGER_DEBUG); - if (DBM::is_result($r)) { - Config::set("system", "post_update_version", 1198); - logger("Done", LOGGER_DEBUG); - return true; - } - return false; - } - - logger("Query done", LOGGER_DEBUG); - - $item_arr = []; - foreach ($r AS $item) { - $index = $item["author-link"]."-".$item["owner-link"]."-".$item["uid"]; - $item_arr[$index] = ["author-link" => $item["author-link"], - "owner-link" => $item["owner-link"], - "uid" => $item["uid"]]; - } - - // Set the "gcontact-id" in the item table and add a new gcontact entry if needed - foreach ($item_arr AS $item) { - $author_id = Contact::getIdForURL($item["author-link"], 0); - $owner_id = Contact::getIdForURL($item["owner-link"], 0); - - if ($author_id == 0) - $author_id = -1; - - if ($owner_id == 0) - $owner_id = -1; - - q("UPDATE `item` SET `author-id` = %d, `owner-id` = %d - WHERE `uid` = %d AND `author-link` = '%s' AND `owner-link` = '%s' - AND `author-id` = 0 AND `owner-id` = 0", - intval($author_id), intval($owner_id), intval($item["uid"]), - dbesc($item["author-link"]), dbesc($item["owner-link"])); - } - - logger("Updated items", LOGGER_DEBUG); - return false; -} - -/** - * @brief update the "last-item" field in the "self" contact - * - * This field avoids cost intensive calls in the admin panel and in "nodeinfo" - * - * @return bool "true" when the job is done - */ -function post_update_1206() { - // Was the script completed? - if (Config::get("system", "post_update_version") >= 1206) - return true; - - logger("Start", LOGGER_DEBUG); - $r = q("SELECT `contact`.`id`, `contact`.`last-item`, - (SELECT MAX(`changed`) FROM `item` USE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` - FROM `user` - INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`"); - - if (!DBM::is_result($r)) { - return false; - } - foreach ($r AS $user) { - if (!empty($user["lastitem_date"]) && ($user["lastitem_date"] > $user["last-item"])) { - q("UPDATE `contact` SET `last-item` = '%s' WHERE `id` = %d", - dbesc($user["lastitem_date"]), - intval($user["id"])); - } - } - - Config::set("system", "post_update_version", 1206); - logger("Done", LOGGER_DEBUG); - return true; -}