From 2424cb78d1abc9e0527f68b90b03556b1fb61cd5 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 31 Jan 2017 19:39:09 +0000 Subject: [PATCH] Only process feed and dfrn items when they aren't already stored --- include/dfrn.php | 60 ++++++++++++++++++++++++++-------------------- include/feed.php | 42 ++++++++++++++++---------------- include/poller.php | 10 ++++++++ 3 files changed, 65 insertions(+), 47 deletions(-) diff --git a/include/dfrn.php b/include/dfrn.php index 702fbb15fe..903db9b601 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -391,7 +391,7 @@ class dfrn { * * @return object XML root object */ - private function add_header($doc, $owner, $authorelement, $alternatelink = "", $public = false) { + private static function add_header($doc, $owner, $authorelement, $alternatelink = "", $public = false) { if ($alternatelink == "") $alternatelink = $owner['url']; @@ -462,7 +462,7 @@ class dfrn { * * @return object XML author object */ - private function add_author($doc, $owner, $authorelement, $public) { + private static function add_author($doc, $owner, $authorelement, $public) { // Is the profile hidden or shouldn't be published in the net? Then add the "hide" element $r = q("SELECT `id` FROM `profile` INNER JOIN `user` ON `user`.`uid` = `profile`.`uid` @@ -591,7 +591,7 @@ class dfrn { * * @return object XML author object */ - private function add_entry_author($doc, $element, $contact_url, $item) { + private static function add_entry_author($doc, $element, $contact_url, $item) { $contact = get_contact_details_by_url($contact_url, $item["uid"]); @@ -631,7 +631,7 @@ class dfrn { * * @return object XML activity object */ - private function create_activity($doc, $element, $activity) { + private static function create_activity($doc, $element, $activity) { if($activity) { $entry = $doc->createElement($element); @@ -685,7 +685,7 @@ class dfrn { * * @return object XML attachment object */ - private function get_attachment($doc, $root, $item) { + private static function get_attachment($doc, $root, $item) { $arr = explode('[/attach],',$item['attach']); if(count($arr)) { foreach($arr as $r) { @@ -720,7 +720,7 @@ class dfrn { * * @return object XML entry object */ - private function entry($doc, $type, $item, $owner, $comment = false, $cid = 0) { + private static function entry($doc, $type, $item, $owner, $comment = false, $cid = 0) { $mentioned = array(); @@ -1103,7 +1103,7 @@ class dfrn { * @param string $birthday Birthday of the contact * */ - private function birthday_event($contact, $birthday) { + private static function birthday_event($contact, $birthday) { // Check for duplicates $r = q("SELECT `id` FROM `event` WHERE `uid` = %d AND `cid` = %d AND `start` = '%s' AND `type` = '%s' LIMIT 1", @@ -1146,7 +1146,7 @@ class dfrn { * * @return Returns an array with relevant data of the author */ - private function fetchauthor($xpath, $context, $importer, $element, $onlyfetch, $xml = "") { + private static function fetchauthor($xpath, $context, $importer, $element, $onlyfetch, $xml = "") { $author = array(); $author["name"] = $xpath->evaluate($element."/atom:name/text()", $context)->item(0)->nodeValue; @@ -1358,7 +1358,7 @@ class dfrn { * * @return string XML string */ - private function transform_activity($xpath, $activity, $element) { + private static function transform_activity($xpath, $activity, $element) { if (!is_object($activity)) return ""; @@ -1403,7 +1403,7 @@ class dfrn { * @param object $mail mail elements * @param array $importer Record of the importer user mixed with contact of the content */ - private function process_mail($xpath, $mail, $importer) { + private static function process_mail($xpath, $mail, $importer) { logger("Processing mails"); @@ -1454,7 +1454,7 @@ class dfrn { * @param object $suggestion suggestion elements * @param array $importer Record of the importer user mixed with contact of the content */ - private function process_suggestion($xpath, $suggestion, $importer) { + private static function process_suggestion($xpath, $suggestion, $importer) { $a = get_app(); logger("Processing suggestions"); @@ -1556,7 +1556,7 @@ class dfrn { * @param object $relocation relocation elements * @param array $importer Record of the importer user mixed with contact of the content */ - private function process_relocation($xpath, $relocation, $importer) { + private static function process_relocation($xpath, $relocation, $importer) { logger("Processing relocations"); @@ -1685,7 +1685,7 @@ class dfrn { * @param array $importer Record of the importer user mixed with contact of the content * @param int $entrytype Is it a toplevel entry, a comment or a relayed comment? */ - private function update_content($current, $item, $importer, $entrytype) { + private static function update_content($current, $item, $importer, $entrytype) { $changed = false; if (edited_timestamp_is_newer($current, $item)) { @@ -1737,7 +1737,7 @@ class dfrn { * * @return int Is it a toplevel entry, a comment or a relayed comment? */ - private function get_entry_type($importer, $item) { + private static function get_entry_type($importer, $item) { if ($item["parent-uri"] != $item["uri"]) { $community = false; @@ -1803,7 +1803,7 @@ class dfrn { * @param array $importer Record of the importer user mixed with contact of the content * @param int $posted_id The record number of item record that was just posted */ - private function do_poke($item, $importer, $posted_id) { + private static function do_poke($item, $importer, $posted_id) { $verb = urldecode(substr($item["verb"],strpos($item["verb"], "#")+1)); if(!$verb) return; @@ -1858,7 +1858,7 @@ class dfrn { * * @return bool Should the processing of the entries be continued? */ - private function process_verbs($entrytype, $importer, &$item, &$is_like) { + private static function process_verbs($entrytype, $importer, &$item, &$is_like) { logger("Process verb ".$item["verb"]." and object-type ".$item["object-type"]." for entrytype ".$entrytype, LOGGER_DEBUG); @@ -1958,7 +1958,7 @@ class dfrn { * @param object $links link elements * @param array $item the item record */ - private function parse_links($links, &$item) { + private static function parse_links($links, &$item) { $rel = ""; $href = ""; $type = ""; @@ -2001,7 +2001,7 @@ class dfrn { * @param object $entry entry elements * @param array $importer Record of the importer user mixed with contact of the content */ - private function process_entry($header, $xpath, $entry, $importer) { + private static function process_entry($header, $xpath, $entry, $importer) { logger("Processing entries"); @@ -2010,6 +2010,20 @@ class dfrn { // Get the uri $item["uri"] = $xpath->query("atom:id/text()", $entry)->item(0)->nodeValue; + $item["edited"] = $xpath->query("atom:updated/text()", $entry)->item(0)->nodeValue; + + $current = q("SELECT `id`, `uid`, `last-child`, `edited`, `body` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1", + dbesc($item["uri"]), + intval($importer["importer_uid"]) + ); + + // Is there an existing item? + if (dbm::is_result($current) AND edited_timestamp_is_newer($current[0], $item) AND + (datetime_convert("UTC","UTC",$item["edited"]) < $current[0]["edited"])) { + logger("Item ".$item["uri"]." already existed.", LOGGER_DEBUG); + return; + } + // Fetch the owner $owner = self::fetchauthor($xpath, $entry, $importer, "dfrn:owner", true); @@ -2027,7 +2041,6 @@ class dfrn { $item["title"] = $xpath->query("atom:title/text()", $entry)->item(0)->nodeValue; $item["created"] = $xpath->query("atom:published/text()", $entry)->item(0)->nodeValue; - $item["edited"] = $xpath->query("atom:updated/text()", $entry)->item(0)->nodeValue; $item["body"] = $xpath->query("dfrn:env/text()", $entry)->item(0)->nodeValue; $item["body"] = str_replace(array(' ',"\t","\r","\n"), array('','','',''),$item["body"]); @@ -2215,18 +2228,13 @@ class dfrn { } } - $r = q("SELECT `id`, `uid`, `last-child`, `edited`, `body` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1", - dbesc($item["uri"]), - intval($importer["importer_uid"]) - ); - if (!self::process_verbs($entrytype, $importer, $item, $is_like)) { logger("Exiting because 'process_verbs' told us so", LOGGER_DEBUG); return; } // Update content if 'updated' changes - if (dbm::is_result($r)) { + if (dbm::is_result($current)) { if (self::update_content($r[0], $item, $importer, $entrytype)) logger("Item ".$item["uri"]." was updated.", LOGGER_DEBUG); else @@ -2311,7 +2319,7 @@ class dfrn { * @param object $deletion deletion elements * @param array $importer Record of the importer user mixed with contact of the content */ - private function process_deletion($xpath, $deletion, $importer) { + private static function process_deletion($xpath, $deletion, $importer) { logger("Processing deletions"); diff --git a/include/feed.php b/include/feed.php index 579ff7caae..e0fef50dbb 100644 --- a/include/feed.php +++ b/include/feed.php @@ -177,18 +177,6 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) { foreach (array_reverse($entrylist) AS $entry) { $item = array_merge($header, $author); - $item["title"] = $xpath->evaluate('atom:title/text()', $entry)->item(0)->nodeValue; - - if ($item["title"] == "") - $item["title"] = $xpath->evaluate('title/text()', $entry)->item(0)->nodeValue; - - if ($item["title"] == "") - $item["title"] = $xpath->evaluate('rss:title/text()', $entry)->item(0)->nodeValue; - - $alternate = $xpath->query("atom:link[@rel='alternate']", $entry)->item(0)->attributes; - if (!is_object($alternate)) - $alternate = $xpath->query("atom:link", $entry)->item(0)->attributes; - if (is_object($alternate)) foreach($alternate AS $attributes) if ($attributes->name == "href") @@ -212,6 +200,27 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) { $item["parent-uri"] = $item["uri"]; + if (!$simulate) { + $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s', '%s')", + intval($importer["uid"]), dbesc($item["uri"]), dbesc(NETWORK_FEED), dbesc(NETWORK_DFRN)); + if ($r) { + logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$r[0]["id"], LOGGER_DEBUG); + continue; + } + } + + $item["title"] = $xpath->evaluate('atom:title/text()', $entry)->item(0)->nodeValue; + + if ($item["title"] == "") + $item["title"] = $xpath->evaluate('title/text()', $entry)->item(0)->nodeValue; + + if ($item["title"] == "") + $item["title"] = $xpath->evaluate('rss:title/text()', $entry)->item(0)->nodeValue; + + $alternate = $xpath->query("atom:link[@rel='alternate']", $entry)->item(0)->attributes; + if (!is_object($alternate)) + $alternate = $xpath->query("atom:link", $entry)->item(0)->attributes; + $published = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue; if ($published == "") @@ -250,15 +259,6 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) { if ($creator != "") $item["author-name"] = $creator; - if (!$simulate) { - $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s', '%s')", - intval($importer["uid"]), dbesc($item["uri"]), dbesc(NETWORK_FEED), dbesc(NETWORK_DFRN)); - if ($r) { - logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$r[0]["id"], LOGGER_DEBUG); - continue; - } - } - /// @TODO ? // Ausland // diff --git a/include/poller.php b/include/poller.php index e8bfb88389..16198580f3 100644 --- a/include/poller.php +++ b/include/poller.php @@ -206,6 +206,16 @@ function poller_exec_function($queue, $funcname, $argv) { $duration = number_format(microtime(true) - $stamp, 3); + if ($duration > 3600) { + logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 1 hour (".round($duration/60, 3).")", LOGGER_DEBUG); + } elseif ($duration > 600) { + logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 10 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); + } elseif ($duration > 300) { + logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 5 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); + } elseif ($duration > 120) { + logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); + } + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); // Write down the performance values into the log