From 4fb059095f7ed058af0f77af5f3a68a401e71bc6 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 21 Dec 2014 02:03:06 +0100 Subject: [PATCH 1/2] Performance stuff: Caching for getsiteinfo function, check before running onepoll.php --- include/items.php | 7 ++++++- include/onepoll.php | 16 +++++++++------- include/poller.php | 16 +++++++++++----- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/include/items.php b/include/items.php index bd1f48c5e3..22ae2f60a7 100644 --- a/include/items.php +++ b/include/items.php @@ -948,7 +948,12 @@ function add_page_info_data($data) { function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") { require_once("mod/parse_url.php"); - $data = parseurl_getsiteinfo($url, true); + $data = Cache::get("parse_url:".$url); + if (is_null($data)){ + $data = parseurl_getsiteinfo($url, true); + Cache::set("parse_url:".$url,serialize($data)); + } else + $data = unserialize($data); if ($photo != "") $data["images"][0]["src"] = $photo; diff --git a/include/onepoll.php b/include/onepoll.php index 9052937fdc..bb5b8905ab 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -15,7 +15,7 @@ function onepoll_run(&$argv, &$argc){ if(is_null($a)) { $a = new App; } - + if(is_null($db)) { @include(".htconfig.php"); require_once("include/dba.php"); @@ -57,28 +57,30 @@ function onepoll_run(&$argv, &$argc){ return; } - // Test $lockpath = get_lockpath(); if ($lockpath != '') { $pidfile = new pidfile($lockpath, 'onepoll'.$contact_id); - if($pidfile->is_already_running()) { + if ($pidfile->is_already_running()) { logger("onepoll: Already running for contact ".$contact_id); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("killed stale process"); + } exit; } } - $d = datetime_convert(); // Only poll from those with suitable relationships, - // and which have a polling address and ignore Diaspora since + // and which have a polling address and ignore Diaspora since // we are unable to match those posts with a Diaspora GUID and prevent duplicates. - $contacts = q("SELECT `contact`.* FROM `contact` + $contacts = q("SELECT `contact`.* FROM `contact` WHERE ( `rel` = %d OR `rel` = %d ) AND `poll` != '' AND NOT `network` IN ( '%s', '%s', '%s' ) AND `contact`.`id` = %d - AND `self` = 0 AND `contact`.`blocked` = 0 AND `contact`.`readonly` = 0 + AND `self` = 0 AND `contact`.`blocked` = 0 AND `contact`.`readonly` = 0 AND `contact`.`archive` = 0 LIMIT 1", intval(CONTACT_IS_SHARING), intval(CONTACT_IS_FRIEND), diff --git a/include/poller.php b/include/poller.php index e94ab8746f..0edda8170a 100644 --- a/include/poller.php +++ b/include/poller.php @@ -240,14 +240,14 @@ function poller_run(&$argv, &$argc){ // We should be getting everything via a hub. But just to be sure, let's check once a day. // (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately) // This also lets us update our subscription to the hub, and add or replace hubs in case it - // changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'. + // changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'. if($contact['subhub']) { $poll_interval = get_config('system','pushpoll_frequency'); $contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3); $hub_update = false; - + if((datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 day")) || $force) $hub_update = true; } @@ -256,13 +256,13 @@ function poller_run(&$argv, &$argc){ /** * Based on $contact['priority'], should we poll this site now? Or later? - */ + */ switch ($contact['priority']) { case 5: if(datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 month")) $update = true; - break; + break; case 4: if(datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 week")) $update = true; @@ -285,7 +285,13 @@ function poller_run(&$argv, &$argc){ continue; } - proc_run('php','include/onepoll.php',$contact['id']); + // Don't run onepoll.php if the contact isn't pollable + // This check also is inside the onepoll.php - but this will reduce the load + if (in_array($contact["rel"], array(CONTACT_IS_SHARING, CONTACT_IS_FRIEND)) AND ($contact["poll"] != "") + AND !in_array($contact['network'], array(NETWORK_DIASPORA, NETWORK_FACEBOOK, NETWORK_PUMPIO, NETWORK_TWITTER, NETWORK_APPNET)) + AND !$contact["self"] AND !$contact["blocked"] AND !$contact["readonly"] AND !$contact["archive"]) + proc_run('php','include/onepoll.php',$contact['id']); + if($interval) @time_sleep_until(microtime(true) + (float) $interval); } From 2c5281cbf1c1334151f6d661e484532e0b8d6c7f Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 21 Dec 2014 02:10:49 +0100 Subject: [PATCH 2/2] reverted a change that wasn't intended to be published yet/Delete thread was missing at Diaspora items. --- include/diaspora.php | 2 + include/threads.php | 109 ++++--------------------------------------- mod/community.php | 25 +--------- 3 files changed, 12 insertions(+), 124 deletions(-) diff --git a/include/diaspora.php b/include/diaspora.php index 8b85e7b955..99288b773c 100755 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -2028,6 +2028,7 @@ function diaspora_retraction($importer,$xml) { dbesc(datetime_convert()), intval($r[0]['id']) ); + delete_thread($r[0]['id']); } } } @@ -2100,6 +2101,7 @@ function diaspora_signed_retraction($importer,$xml,$msg) { dbesc(datetime_convert()), intval($r[0]['id']) ); + delete_thread($r[0]['id']); // Now check if the retraction needs to be relayed by us // diff --git a/include/threads.php b/include/threads.php index 0ad1d396df..6b1c4342f7 100644 --- a/include/threads.php +++ b/include/threads.php @@ -13,51 +13,9 @@ function add_thread($itemid) { .implode("`, `", array_keys($item)) ."`) VALUES ('" .implode("', '", array_values($item)) - ."')"); + ."')" ); logger("add_thread: Add thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG); - - // Store a shadow copy of public items for displaying a global community page? - if (!get_config('system', 'global_community')) - return; - - // is it already a copy? - if (($itemid == 0) OR ($item['uid'] == 0)) - return; - - // Check, if hide-friends is activated - then don't do a shadow entry - $r = q("SELECT `hide-friends` FROM `profile` WHERE `is-default` AND `uid` = %d AND NOT `hide-friends`", - $item['uid']); - if (!count($r)) - return; - - // Only add a shadow, if the profile isn't hidden - $r = q("SELECT `uid` FROM `user` where `uid` = %d AND NOT `hidewall`", $item['uid']); - if (!count($r)) - return; - - $item = q("SELECT * FROM `item` WHERE `id` = %d", - intval($itemid)); - - if (count($item) AND ($item[0]["visible"] == 1) AND ($item[0]["deleted"] == 0) AND - (($item[0]["id"] == $item[0]["parent"]) OR ($item[0]["parent"] == 0)) AND - ($item[0]["moderated"] == 0) AND ($item[0]["allow_cid"] == '') AND ($item[0]["allow_gid"] == '') AND - ($item[0]["deny_cid"] == '') AND ($item[0]["deny_gid"] == '') AND ($item[0]["private"] == 0)) { - - $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = 0 LIMIT 1", - dbesc($item['uri'])); - - if (!$r) { - // Preparing public shadow (removing user specific data) - require_once("include/items.php"); - unset($item[0]['id']); - $item[0]['uid'] = 0; - $item[0]['contact-id'] = 0; - - $public_shadow = item_store($item[0],false); - logger("add_thread: Stored public shadow for post ".$itemid." under id ".$public_shadow, LOGGER_DEBUG); - } - } } function update_thread_uri($itemuri, $uid) { @@ -69,7 +27,7 @@ function update_thread_uri($itemuri, $uid) { } function update_thread($itemid, $setmention = false) { - $items = q("SELECT `uid`, `uri`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`, `moderated`, `visible`, `spam`, `starred`, `bookmark`, `contact-id`, + $items = q("SELECT `uid`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`, `moderated`, `visible`, `spam`, `starred`, `bookmark`, `contact-id`, `deleted`, `origin`, `forum_mode`, `network` FROM `item` WHERE `id` = %d AND (`parent` = %d OR `parent` = 0) LIMIT 1", intval($itemid), intval($itemid)); if (!$items) @@ -82,50 +40,16 @@ function update_thread($itemid, $setmention = false) { $sql = ""; - foreach ($item AS $field => $data) - if ($field != "uri") { - if ($sql != "") - $sql .= ", "; + foreach ($item AS $field => $data) { + if ($sql != "") + $sql .= ", "; - $sql .= "`".$field."` = '".dbesc($data)."'"; - } + $sql .= "`".$field."` = '".$data."'"; + } - $result = q("UPDATE `thread` SET ".$sql." WHERE `iid` = %d", intval($itemid)); + $result = q("UPDATE `thread` SET ".$sql." WHERE `iid` = %d", $itemid); logger("update_thread: Update thread for item ".$itemid." - ".print_r($result, true)." ".print_r($item, true), LOGGER_DEBUG); - - // Updating a shadow item entry - $items = q("SELECT `id`, `title`, `body`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`, - `moderated`, `visible`, `spam`, `starred`, `bookmark`, `deleted`, `origin`, `forum_mode`, `network` - FROM `item` WHERE `uri` = '%s' AND `uid` = 0 LIMIT 1", dbesc($item["uri"])); - - if (!$items) - return; - - $item = $items[0]; - - $result = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `network` = '%s' WHERE `id` = %d", - dbesc($item["title"]), - dbesc($item["body"]), - dbesc($item["network"]), - intval($item["id"]) - ); - logger("update_thread: Updating public shadow for post ".$item["id"]." Result: ".print_r($result, true), LOGGER_DEBUG); - - /* - $sql = ""; - - foreach ($item AS $field => $data) - if ($field != "id") { - if ($sql != "") - $sql .= ", "; - - $sql .= "`".$field."` = '".dbesc($data)."'"; - } - //logger("update_thread: Updating public shadow for post ".$item["id"]." SQL: ".$sql, LOGGER_DEBUG); - $result = q("UPDATE `item` SET ".$sql." WHERE `id` = %d", intval($item["id"])); - logger("update_thread: Updating public shadow for post ".$item["id"]." Result: ".print_r($result, true), LOGGER_DEBUG); - */ } function delete_thread_uri($itemuri, $uid) { @@ -137,28 +61,13 @@ function delete_thread_uri($itemuri, $uid) { } function delete_thread($itemid) { - $item = q("SELECT `uri`, `uid` FROM `thread` WHERE `iid` = %d", intval($itemid)); - $result = q("DELETE FROM `thread` WHERE `iid` = %d", intval($itemid)); logger("delete_thread: Deleted thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG); - - if (count($item)) { - $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND NOT (`uid` IN (%d, 0))", - dbesc($item["uri"]), - intval($item["uid"]) - ); - if (!count($r)) { - $r = q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = 0)", - dbesc($item["uri"]) - ); - logger("delete_thread: Deleted shadow for item ".$item["uri"]." - ".print_r($result, true), LOGGER_DEBUG); - } - } } function update_threads() { - global $db; + global $db; logger("update_threads: start"); diff --git a/mod/community.php b/mod/community.php index 30eb6ebb03..8d23c4af28 100644 --- a/mod/community.php +++ b/mod/community.php @@ -114,10 +114,6 @@ function community_content(&$a, $update = 0) { } function community_getitems($start, $itemspage) { - // Work in progress - if (get_config('system', 'global_community')) - return(community_getpublicitems($start, $itemspage)); - $r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, `contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`, `contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`, @@ -129,7 +125,7 @@ function community_getitems($start, $itemspage) { AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' INNER JOIN `contact` ON `contact`.`id` = `thread`.`contact-id` - AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0 AND (`contact`.`self` OR `contact`.`remote_self`) + AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0 AND `contact`.`self` WHERE `thread`.`visible` = 1 AND `thread`.`deleted` = 0 and `thread`.`moderated` = 0 AND `thread`.`private` = 0 AND `thread`.`wall` = 1 ORDER BY `thread`.`received` DESC LIMIT %d, %d ", @@ -140,22 +136,3 @@ function community_getitems($start, $itemspage) { return($r); } - -function community_getpublicitems($start, $itemspage) { - $r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, - `author-name` AS `name`, `owner-avatar` AS `photo`, - `owner-link` AS `url`, `owner-avatar` AS `thumb` - FROM `item` WHERE `item`.`uid` = 0 AND `network` IN ('%s', '%s', '%s') - AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' - AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' - ORDER BY `item`.`received` DESC LIMIT %d, %d", - dbesc(NETWORK_DFRN), - dbesc(NETWORK_DIASPORA), - dbesc(NETWORK_OSTATUS), - intval($start), - intval($itemspage) - ); - - return($r); - -}