From beb1e040696ae53b58aebdd2f639b0eb90c639fd Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 3 Apr 2016 13:48:31 +0200 Subject: [PATCH 1/4] New field in item table ("shadow") that indicates if there is a shadow entry --- boot.php | 8 +-- include/Contact.php | 54 ----------------- include/cron.php | 5 +- include/dbstructure.php | 1 + include/post_update.php | 129 ++++++++++++++++++++++++++++++++++++++++ include/threads.php | 8 ++- update.php | 2 +- 7 files changed, 145 insertions(+), 62 deletions(-) create mode 100644 include/post_update.php diff --git a/boot.php b/boot.php index a70c7bf616..13cc2aaf5b 100644 --- a/boot.php +++ b/boot.php @@ -6,11 +6,11 @@ /** * Friendica - * + * * Friendica is a communications platform for integrated social communications * utilising decentralised communications and linkage to several indie social * projects - as well as popular mainstream providers. - * + * * Our mission is to free our friends and families from the clutches of * data-harvesting corporations, and pave the way to a future where social * communications are free and open and flow between alternate providers as @@ -18,7 +18,7 @@ */ require_once('include/autoloader.php'); - + require_once('include/config.php'); require_once('include/network.php'); require_once('include/plugin.php'); @@ -38,7 +38,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Asparagus'); define ( 'FRIENDICA_VERSION', '3.5-dev' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1194 ); +define ( 'DB_UPDATE_VERSION', 1195 ); /** * @brief Constant with a HTML line break. diff --git a/include/Contact.php b/include/Contact.php index d76c8f826c..79a14ab581 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -555,60 +555,6 @@ function posts_from_gcontact($a, $gcontact_id) { return $o; } -/** - * @brief set the gcontact-id in all item entries - * - * This job has to be started multiple times until all entries are set. - * It isn't started in the update function since it would consume too much time and can be done in the background. - */ -function item_set_gcontact() { - define ('POST_UPDATE_VERSION', 1192); - - // Was the script completed? - if (get_config("system", "post_update_version") >= POST_UPDATE_VERSION) - return; - - // Check if the first step is done (Setting "gcontact-id" in the item table) - $r = q("SELECT `author-link`, `author-name`, `author-avatar`, `uid`, `network` FROM `item` WHERE `gcontact-id` = 0 LIMIT 1000"); - if (!$r) { - // Are there unfinished entries in the thread table? - $r = q("SELECT COUNT(*) AS `total` FROM `thread` - INNER JOIN `item` ON `item`.`id` =`thread`.`iid` - WHERE `thread`.`gcontact-id` = 0 AND - (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); - - if ($r AND ($r[0]["total"] == 0)) { - set_config("system", "post_update_version", POST_UPDATE_VERSION); - return false; - } - - // Update the thread table from the item table - q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid` - SET `thread`.`gcontact-id` = `item`.`gcontact-id` - WHERE `thread`.`gcontact-id` = 0 AND - (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); - - return false; - } - - $item_arr = array(); - foreach ($r AS $item) { - $index = $item["author-link"]."-".$item["uid"]; - $item_arr[$index] = array("author-link" => $item["author-link"], - "uid" => $item["uid"], - "network" => $item["network"]); - } - - // Set the "gcontact-id" in the item table and add a new gcontact entry if needed - foreach($item_arr AS $item) { - $gcontact_id = get_gcontact_id(array("url" => $item['author-link'], "network" => $item['network'], - "photo" => $item['author-avatar'], "name" => $item['author-name'])); - q("UPDATE `item` SET `gcontact-id` = %d WHERE `uid` = %d AND `author-link` = '%s' AND `gcontact-id` = 0", - intval($gcontact_id), intval($item["uid"]), dbesc($item["author-link"])); - } - return true; -} - /** * @brief Returns posts from a given contact * diff --git a/include/cron.php b/include/cron.php index e2f4102804..4eb76f1ce2 100644 --- a/include/cron.php +++ b/include/cron.php @@ -35,6 +35,7 @@ function cron_run(&$argv, &$argc){ require_once('include/email.php'); require_once('include/socgraph.php'); require_once('mod/nodeinfo.php'); + require_once('include/post_update.php'); load_config('config'); load_config('system'); @@ -106,8 +107,8 @@ function cron_run(&$argv, &$argc){ // Check every conversation ostatus::check_conversations(false); - // Set the gcontact-id in the item table if missing - item_set_gcontact(); + // Do post update functions + post_update(); // update nodeinfo data nodeinfo_cron(); diff --git a/include/dbstructure.php b/include/dbstructure.php index e34e409023..33e0c7dc8c 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -783,6 +783,7 @@ function db_definition() { "parent-uri" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "extid" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "thr-parent" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "shadow" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"), "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "edited" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "commented" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), diff --git a/include/post_update.php b/include/post_update.php new file mode 100644 index 0000000000..2143ac3d6f --- /dev/null +++ b/include/post_update.php @@ -0,0 +1,129 @@ += 1192) + return true; + + // Check if the first step is done (Setting "gcontact-id" in the item table) + $r = q("SELECT `author-link`, `author-name`, `author-avatar`, `uid`, `network` FROM `item` WHERE `gcontact-id` = 0 LIMIT 1000"); + if (!$r) { + // Are there unfinished entries in the thread table? + $r = q("SELECT COUNT(*) AS `total` FROM `thread` + INNER JOIN `item` ON `item`.`id` =`thread`.`iid` + WHERE `thread`.`gcontact-id` = 0 AND + (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); + + if ($r AND ($r[0]["total"] == 0)) { + set_config("system", "post_update_version", 1192); + return true; + } + + // Update the thread table from the item table + q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid` + SET `thread`.`gcontact-id` = `item`.`gcontact-id` + WHERE `thread`.`gcontact-id` = 0 AND + (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); + + return false; + } + + $item_arr = array(); + foreach ($r AS $item) { + $index = $item["author-link"]."-".$item["uid"]; + $item_arr[$index] = array("author-link" => $item["author-link"], + "uid" => $item["uid"], + "network" => $item["network"]); + } + + // Set the "gcontact-id" in the item table and add a new gcontact entry if needed + foreach($item_arr AS $item) { + $gcontact_id = get_gcontact_id(array("url" => $item['author-link'], "network" => $item['network'], + "photo" => $item['author-avatar'], "name" => $item['author-name'])); + q("UPDATE `item` SET `gcontact-id` = %d WHERE `uid` = %d AND `author-link` = '%s' AND `gcontact-id` = 0", + intval($gcontact_id), intval($item["uid"]), dbesc($item["author-link"])); + } + return false; +} + +/** + * @brief Updates the "shadow" field in the item table + * + * @return bool "true" when the job is done + */ +function post_update_1195() { + + // Was the script completed? + if (get_config("system", "post_update_version") >= 1195) + return true; + + $end_id = get_config("system", "post_update_1195_end"); + if (!$end_id) { + $r = q("SELECT `id` FROM `item` WHERE `uid` != 0 ORDER BY `id` DESC LIMIT 1"); + if ($r) { + set_config("system", "post_update_1195_end", $r[0]["id"]); + $end_id = get_config("system", "post_update_1195_end"); + } + } + + $start_id = get_config("system", "post_update_1195_start"); + + $query1 = "SELECT `item`.`id` FROM `item` "; + + $query2 = "INNER JOIN `item` AS `shadow` ON `item`.`uri` = `shadow`.`uri` AND `shadow`.`uid` = 0 "; + + $query3 = "WHERE `item`.`uid` != 0 AND `item`.`id` >= %d AND `item`.`id` <= %d + AND `item`.`visible` AND NOT `item`.`private` + AND NOT `item`.`deleted` AND NOT `item`.`moderated` + AND `item`.`network` IN ('%s', '%s', '%s', '') + AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' + AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' + AND `item`.`shadow` = 0"; + + $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 1", + intval($start_id), intval($end_id), + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); + if (!$r) { + set_config("system", "post_update_version", 1195); + return true; + } else { + set_config("system", "post_update_1195_start", $r[0]["id"]); + $start_id = get_config("system", "post_update_1195_start"); + } + + + $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 10000,1", + intval($start_id), intval($end_id), + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); + if ($r) + $pos_id = $r[0]["id"]; + else + $pos_id = $end_id; + + logger("Progress: Start: ".$start_id." position: ".$pos_id." end: ".$end_id, LOGGER_DEBUG); + + $r = q("UPDATE `item` ".$query2." SET `item`.`shadow` = `shadow`.`id` ".$query3, + intval($start_id), intval($pos_id), + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); +} +?> diff --git a/include/threads.php b/include/threads.php index 0320eaa018..21fdb0df34 100644 --- a/include/threads.php +++ b/include/threads.php @@ -65,13 +65,19 @@ function add_thread($itemid, $onlyshadow = false) { require_once("include/Contact.php"); unset($item[0]['id']); + unset($item[0]['shadow']); $item[0]['uid'] = 0; $item[0]['origin'] = 0; $item[0]['contact-id'] = get_contact($item[0]['author-link'], 0); $public_shadow = item_store($item[0], false, false, true); logger("add_thread: Stored public shadow for post ".$itemid." under id ".$public_shadow, LOGGER_DEBUG); - } + } else + $public_shadow = $r[0]["id"]; + + if ($public_shadow > 0) + q("UPDATE `item` SET `shadow` = %d WHERE `id` = %d", intval($public_shadow), intval($itemid)); + } } diff --git a/update.php b/update.php index 0689aa19b3..dad94f271b 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ Date: Sun, 3 Apr 2016 16:36:05 +0200 Subject: [PATCH 2/4] Post update ist now done. --- boot.php | 2 +- include/dbstructure.php | 1 - include/items.php | 4 ++-- include/post_update.php | 40 ++++++++++++++++++++++++++-------------- mod/search.php | 5 ++--- update.php | 2 +- 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/boot.php b/boot.php index 13cc2aaf5b..58b4bc0983 100644 --- a/boot.php +++ b/boot.php @@ -38,7 +38,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Asparagus'); define ( 'FRIENDICA_VERSION', '3.5-dev' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1195 ); +define ( 'DB_UPDATE_VERSION', 1194 ); /** * @brief Constant with a HTML line break. diff --git a/include/dbstructure.php b/include/dbstructure.php index 33e0c7dc8c..e34e409023 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -783,7 +783,6 @@ function db_definition() { "parent-uri" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "extid" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "thr-parent" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), - "shadow" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"), "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "edited" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "commented" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), diff --git a/include/items.php b/include/items.php index 233d72d133..4627b10ca2 100644 --- a/include/items.php +++ b/include/items.php @@ -707,9 +707,9 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa if ($arr["uid"] == 0) { $arr["global"] = true; - q("UPDATE `item` SET `global` = 1 WHERE `guid` = '%s'", dbesc($arr["guid"])); + q("UPDATE `item` SET `global` = 1 WHERE `uri` = '%s'", dbesc($arr["uri"])); } else { - $isglobal = q("SELECT `global` FROM `item` WHERE `uid` = 0 AND `guid` = '%s'", dbesc($arr["guid"])); + $isglobal = q("SELECT `global` FROM `item` WHERE `uid` = 0 AND `uri` = '%s'", dbesc($arr["uri"])); $arr["global"] = (count($isglobal) > 0); } diff --git a/include/post_update.php b/include/post_update.php index 2143ac3d6f..2bdfe1f6fd 100644 --- a/include/post_update.php +++ b/include/post_update.php @@ -1,4 +1,8 @@ = 1195) + if (get_config("system", "post_update_version") >= 1194) return true; - $end_id = get_config("system", "post_update_1195_end"); + logger("Start", LOGGER_DEBUG); + + $end_id = get_config("system", "post_update_1194_end"); if (!$end_id) { $r = q("SELECT `id` FROM `item` WHERE `uid` != 0 ORDER BY `id` DESC LIMIT 1"); if ($r) { - set_config("system", "post_update_1195_end", $r[0]["id"]); - $end_id = get_config("system", "post_update_1195_end"); + set_config("system", "post_update_1194_end", $r[0]["id"]); + $end_id = get_config("system", "post_update_1194_end"); } } - $start_id = get_config("system", "post_update_1195_start"); + logger("End ID: ".$end_id, LOGGER_DEBUG); + + $start_id = get_config("system", "post_update_1194_start"); $query1 = "SELECT `item`.`id` FROM `item` "; @@ -98,21 +106,23 @@ function post_update_1195() { AND `item`.`network` IN ('%s', '%s', '%s', '') AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' - AND `item`.`shadow` = 0"; + AND NOT `item`.`global`"; $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 1", intval($start_id), intval($end_id), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); if (!$r) { - set_config("system", "post_update_version", 1195); + set_config("system", "post_update_version", 1194); + logger("Update is done", LOGGER_DEBUG); return true; } else { - set_config("system", "post_update_1195_start", $r[0]["id"]); - $start_id = get_config("system", "post_update_1195_start"); + set_config("system", "post_update_1194_start", $r[0]["id"]); + $start_id = get_config("system", "post_update_1194_start"); } + logger("Start ID: ".$start_id, LOGGER_DEBUG); - $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 10000,1", + $r = q($query1.$query2.$query3." ORDER BY `item`.`id` LIMIT 1000,1", intval($start_id), intval($end_id), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); if ($r) @@ -122,8 +132,10 @@ function post_update_1195() { logger("Progress: Start: ".$start_id." position: ".$pos_id." end: ".$end_id, LOGGER_DEBUG); - $r = q("UPDATE `item` ".$query2." SET `item`.`shadow` = `shadow`.`id` ".$query3, + $r = q("UPDATE `item` ".$query2." SET `item`.`global` = 1 ".$query3, intval($start_id), intval($pos_id), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); + + logger("Done", LOGGER_DEBUG); } ?> diff --git a/mod/search.php b/mod/search.php index 1776a92552..790f577ba6 100644 --- a/mod/search.php +++ b/mod/search.php @@ -217,11 +217,10 @@ function search_content(&$a) { FROM `item` INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND NOT `contact`.`blocked` AND NOT `contact`.`pending` WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated` - AND (`item`.`uid` = 0 OR (`item`.`uid` = %s AND (`item`.`private` OR NOT `item`.`network` IN ('%s', '%s', '%s')))) + AND (`item`.`uid` = 0 OR (`item`.`uid` = %s AND NOT `item`.`global`)) $sql_extra GROUP BY `item`.`uri` ORDER BY `item`.`id` DESC LIMIT %d , %d ", - intval(local_user()), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), - intval($a->pager['start']), intval($a->pager['itemspage'])); + intval(local_user()), intval($a->pager['start']), intval($a->pager['itemspage'])); } if(! count($r)) { diff --git a/update.php b/update.php index dad94f271b..0689aa19b3 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ Date: Sun, 3 Apr 2016 16:38:32 +0200 Subject: [PATCH 3/4] Some reverted stuff --- include/threads.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/include/threads.php b/include/threads.php index 21fdb0df34..0320eaa018 100644 --- a/include/threads.php +++ b/include/threads.php @@ -65,19 +65,13 @@ function add_thread($itemid, $onlyshadow = false) { require_once("include/Contact.php"); unset($item[0]['id']); - unset($item[0]['shadow']); $item[0]['uid'] = 0; $item[0]['origin'] = 0; $item[0]['contact-id'] = get_contact($item[0]['author-link'], 0); $public_shadow = item_store($item[0], false, false, true); logger("add_thread: Stored public shadow for post ".$itemid." under id ".$public_shadow, LOGGER_DEBUG); - } else - $public_shadow = $r[0]["id"]; - - if ($public_shadow > 0) - q("UPDATE `item` SET `shadow` = %d WHERE `id` = %d", intval($public_shadow), intval($itemid)); - + } } } From 3fdaafd54a8a24c5f265eb0dcdc218a51901a4eb Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 3 Apr 2016 18:49:41 +0200 Subject: [PATCH 4/4] Some more documentation --- include/cron.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/cron.php b/include/cron.php index 4eb76f1ce2..00dd500704 100644 --- a/include/cron.php +++ b/include/cron.php @@ -107,7 +107,8 @@ function cron_run(&$argv, &$argc){ // Check every conversation ostatus::check_conversations(false); - // Do post update functions + // Call possible post update functions + // see include/post_update.php for more details post_update(); // update nodeinfo data