bitPickups/friendica_2019-12_sharedHosting_hotFix
1
1
Fork 0
Code Pull requests Activity

Merge pull request #3216 from annando/1701-performance

Browse source 
Better network handling, improved structure
This commit is contained in:
Tobias Diekershoff 2017-03-12 15:40:56 +01:00 committed by GitHub
parent e29d7f19d1 2d2a35af73
commit 1bf089e424
10 changed files with 333 additions and 212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
include/Probe.php
View file

@ -60,11 +60,20 @@ class Probe {
$xrd_timeout = Config::get('system','xrd_timeout', 20); $xrd_timeout = Config::get('system','xrd_timeout', 20);
$redirects = 0; $redirects = 0;
$xml = fetch_url($ssl_url, false, $redirects, $xrd_timeout, "application/xrd+xml"); $ret = z_fetch_url($ssl_url, false, $redirects, array('timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml'));
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return false;
}
$xml = $ret['body'];
$xrd = parse_xml_string($xml, false); $xrd = parse_xml_string($xml, false);
if (!is_object($xrd)) { if (!is_object($xrd)) {
$xml = fetch_url($url, false, $redirects, $xrd_timeout, "application/xrd+xml"); $ret = z_fetch_url($url, false, $redirects, array('timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml'));
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return false;
}
$xml = $ret['body'];
$xrd = parse_xml_string($xml, false); $xrd = parse_xml_string($xml, false);
} }
if (!is_object($xrd)) if (!is_object($xrd))
@ -430,7 +439,12 @@ class Probe {
$xrd_timeout = Config::get('system','xrd_timeout', 20); $xrd_timeout = Config::get('system','xrd_timeout', 20);
$redirects = 0; $redirects = 0;
$data = fetch_url($url, false, $redirects, $xrd_timeout, "application/xrd+xml"); $ret = z_fetch_url($url, false, $redirects, array('timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml'));
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return false;
}
$data = $ret['body'];
$xrd = parse_xml_string($data, false); $xrd = parse_xml_string($data, false);
if (!is_object($xrd)) { if (!is_object($xrd)) {
@ -482,9 +496,14 @@ class Probe {
* @return array noscrape data * @return array noscrape data
*/ */
private function poll_noscrape($noscrape, $data) { private function poll_noscrape($noscrape, $data) {
$content = fetch_url($noscrape); $ret = z_fetch_url($noscrape);
if (!$content) if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return false; return false;
}
$content = $ret['body'];
if (!$content) {
return false;
}
$json = json_decode($content, true); $json = json_decode($content, true);
if (!is_array($json)) if (!is_array($json))
@ -663,10 +682,14 @@ class Probe {
* @return array hcard data * @return array hcard data
*/ */
private function poll_hcard($hcard, $data, $dfrn = false) { private function poll_hcard($hcard, $data, $dfrn = false) {
$ret = z_fetch_url($hcard);
$content = fetch_url($hcard); if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
if (!$content)
return false; return false;
}
$content = $ret['body'];
if (!$content) {
return false;
}
$doc = new DOMDocument(); $doc = new DOMDocument();
if (!@$doc->loadHTML($content)) if (!@$doc->loadHTML($content))
@ -859,8 +882,13 @@ class Probe {
$pubkey = substr($pubkey, strpos($pubkey, ',') + 1); $pubkey = substr($pubkey, strpos($pubkey, ',') + 1);
else else
$pubkey = substr($pubkey, 5); $pubkey = substr($pubkey, 5);
} elseif (normalise_link($pubkey) == 'http://') } elseif (normalise_link($pubkey) == 'http://') {
$pubkey = fetch_url($pubkey); $ret = z_fetch_url($pubkey);
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return false;
}
$pubkey = $ret['body'];
}
$key = explode(".", $pubkey); $key = explode(".", $pubkey);
@ -880,7 +908,11 @@ class Probe {
return false; return false;
// Fetch all additional data from the feed // Fetch all additional data from the feed
$feed = fetch_url($data["poll"]); $ret = z_fetch_url($data["poll"]);
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return false;
}
$feed = $ret['body'];
$feed_data = feed_import($feed,$dummy1,$dummy2, $dummy3, true); $feed_data = feed_import($feed,$dummy1,$dummy2, $dummy3, true);
if (!$feed_data) if (!$feed_data)
return false; return false;
@ -1035,7 +1067,11 @@ class Probe {
* @return array feed data * @return array feed data
*/ */
private function feed($url, $probe = true) { private function feed($url, $probe = true) {
$feed = fetch_url($url); $ret = z_fetch_url($url);
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return false;
}
$feed = $ret['body'];
$feed_data = feed_import($feed, $dummy1, $dummy2, $dummy3, true); $feed_data = feed_import($feed, $dummy1, $dummy2, $dummy3, true);
if (!$feed_data) { if (!$feed_data) {

2
include/delivery.php
View file

@ -398,7 +398,7 @@ function delivery_run(&$argv, &$argc){
logger('notifier: dfrn_delivery to '.$contact["url"].' with guid '.$target_item["guid"].' returns '.$deliver_status); logger('notifier: dfrn_delivery to '.$contact["url"].' with guid '.$target_item["guid"].' returns '.$deliver_status);
if ($deliver_status == (-1)) { if ($deliver_status < 0) {
logger('notifier: delivery failed: queuing message'); logger('notifier: delivery failed: queuing message');
add_to_queue($contact['id'],NETWORK_DFRN,$atom); add_to_queue($contact['id'],NETWORK_DFRN,$atom);

95
include/dfrn.php
View file

@ -391,7 +391,7 @@ class dfrn {
* *
* @return object XML root object * @return object XML root object
*/ */
private function add_header($doc, $owner, $authorelement, $alternatelink = "", $public = false) { private static function add_header($doc, $owner, $authorelement, $alternatelink = "", $public = false) {
if ($alternatelink == "") if ($alternatelink == "")
$alternatelink = $owner['url']; $alternatelink = $owner['url'];
@ -462,7 +462,7 @@ class dfrn {
* *
* @return object XML author object * @return object XML author object
*/ */
private function add_author($doc, $owner, $authorelement, $public) { private static function add_author($doc, $owner, $authorelement, $public) {
// Is the profile hidden or shouldn't be published in the net? Then add the "hide" element // Is the profile hidden or shouldn't be published in the net? Then add the "hide" element
$r = q("SELECT `id` FROM `profile` INNER JOIN `user` ON `user`.`uid` = `profile`.`uid` $r = q("SELECT `id` FROM `profile` INNER JOIN `user` ON `user`.`uid` = `profile`.`uid`
@ -591,7 +591,7 @@ class dfrn {
* *
* @return object XML author object * @return object XML author object
*/ */
private function add_entry_author($doc, $element, $contact_url, $item) { private static function add_entry_author($doc, $element, $contact_url, $item) {
$contact = get_contact_details_by_url($contact_url, $item["uid"]); $contact = get_contact_details_by_url($contact_url, $item["uid"]);
@ -631,7 +631,7 @@ class dfrn {
* *
* @return object XML activity object * @return object XML activity object
*/ */
private function create_activity($doc, $element, $activity) { private static function create_activity($doc, $element, $activity) {
if($activity) { if($activity) {
$entry = $doc->createElement($element); $entry = $doc->createElement($element);
@ -685,7 +685,7 @@ class dfrn {
* *
* @return object XML attachment object * @return object XML attachment object
*/ */
private function get_attachment($doc, $root, $item) { private static function get_attachment($doc, $root, $item) {
$arr = explode('[/attach],',$item['attach']); $arr = explode('[/attach],',$item['attach']);
if(count($arr)) { if(count($arr)) {
foreach($arr as $r) { foreach($arr as $r) {
@ -720,7 +720,7 @@ class dfrn {
* *
* @return object XML entry object * @return object XML entry object
*/ */
private function entry($doc, $type, $item, $owner, $comment = false, $cid = 0) { private static function entry($doc, $type, $item, $owner, $comment = false, $cid = 0) {
$mentioned = array(); $mentioned = array();
@ -913,11 +913,18 @@ class dfrn {
logger('dfrn_deliver: ' . $url); logger('dfrn_deliver: ' . $url);
$xml = fetch_url($url); $ret = z_fetch_url($url);
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return -2; // timed out
}
$xml = $ret['body'];
$curl_stat = $a->get_curl_code(); $curl_stat = $a->get_curl_code();
if(! $curl_stat) if (!$curl_stat) {
return(-1); // timed out return -3; // timed out
}
logger('dfrn_deliver: ' . $xml, LOGGER_DATA); logger('dfrn_deliver: ' . $xml, LOGGER_DATA);
@ -1017,24 +1024,24 @@ class dfrn {
$key = Crypto::createNewRandomKey(); $key = Crypto::createNewRandomKey();
} catch (CryptoTestFailed $ex) { } catch (CryptoTestFailed $ex) {
logger('Cannot safely create a key'); logger('Cannot safely create a key');
return -1; return -4;
} catch (CannotPerformOperation $ex) { } catch (CannotPerformOperation $ex) {
logger('Cannot safely create a key'); logger('Cannot safely create a key');
return -1; return -5;
} }
try { try {
$data = Crypto::encrypt($postvars['data'], $key); $data = Crypto::encrypt($postvars['data'], $key);
} catch (CryptoTestFailed $ex) { } catch (CryptoTestFailed $ex) {
logger('Cannot safely perform encryption'); logger('Cannot safely perform encryption');
return -1; return -6;
} catch (CannotPerformOperation $ex) { } catch (CannotPerformOperation $ex) {
logger('Cannot safely perform encryption'); logger('Cannot safely perform encryption');
return -1; return -7;
} }
break; break;
default: default:
logger("rino: invalid requested verision '$rino_remote_version'"); logger("rino: invalid requested verision '$rino_remote_version'");
return -1; return -8;
} }
$postvars['rino'] = $rino_remote_version; $postvars['rino'] = $rino_remote_version;
@ -1068,16 +1075,18 @@ class dfrn {
logger('dfrn_deliver: ' . "SENDING: " . print_r($postvars,true), LOGGER_DATA); logger('dfrn_deliver: ' . "SENDING: " . print_r($postvars,true), LOGGER_DATA);
$xml = post_url($contact['notify'],$postvars); $xml = post_url($contact['notify'], $postvars);
logger('dfrn_deliver: ' . "RECEIVED: " . $xml, LOGGER_DATA); logger('dfrn_deliver: ' . "RECEIVED: " . $xml, LOGGER_DATA);
$curl_stat = $a->get_curl_code(); $curl_stat = $a->get_curl_code();
if((! $curl_stat) || (! strlen($xml))) if ((!$curl_stat) || (!strlen($xml))) {
return(-1); // timed out return -9; // timed out
}
if(($curl_stat == 503) && (stristr($a->get_curl_headers(),'retry-after'))) if (($curl_stat == 503) && (stristr($a->get_curl_headers(),'retry-after'))) {
return(-1); return -10;
}
if(strpos($xml,'<?xml') === false) { if(strpos($xml,'<?xml') === false) {
logger('dfrn_deliver: phase 2: no valid XML returned'); logger('dfrn_deliver: phase 2: no valid XML returned');
@ -1103,7 +1112,7 @@ class dfrn {
* @param string $birthday Birthday of the contact * @param string $birthday Birthday of the contact
* *
*/ */
private function birthday_event($contact, $birthday) { private static function birthday_event($contact, $birthday) {
// Check for duplicates // Check for duplicates
$r = q("SELECT `id` FROM `event` WHERE `uid` = %d AND `cid` = %d AND `start` = '%s' AND `type` = '%s' LIMIT 1", $r = q("SELECT `id` FROM `event` WHERE `uid` = %d AND `cid` = %d AND `start` = '%s' AND `type` = '%s' LIMIT 1",
@ -1146,7 +1155,7 @@ class dfrn {
* *
* @return Returns an array with relevant data of the author * @return Returns an array with relevant data of the author
*/ */
private function fetchauthor($xpath, $context, $importer, $element, $onlyfetch, $xml = "") { private static function fetchauthor($xpath, $context, $importer, $element, $onlyfetch, $xml = "") {
$author = array(); $author = array();
$author["name"] = $xpath->evaluate($element."/atom:name/text()", $context)->item(0)->nodeValue; $author["name"] = $xpath->evaluate($element."/atom:name/text()", $context)->item(0)->nodeValue;
@ -1358,7 +1367,7 @@ class dfrn {
* *
* @return string XML string * @return string XML string
*/ */
private function transform_activity($xpath, $activity, $element) { private static function transform_activity($xpath, $activity, $element) {
if (!is_object($activity)) if (!is_object($activity))
return ""; return "";
@ -1403,7 +1412,7 @@ class dfrn {
* @param object $mail mail elements * @param object $mail mail elements
* @param array $importer Record of the importer user mixed with contact of the content * @param array $importer Record of the importer user mixed with contact of the content
*/ */
private function process_mail($xpath, $mail, $importer) { private static function process_mail($xpath, $mail, $importer) {
logger("Processing mails"); logger("Processing mails");
@ -1454,7 +1463,7 @@ class dfrn {
* @param object $suggestion suggestion elements * @param object $suggestion suggestion elements
* @param array $importer Record of the importer user mixed with contact of the content * @param array $importer Record of the importer user mixed with contact of the content
*/ */
private function process_suggestion($xpath, $suggestion, $importer) { private static function process_suggestion($xpath, $suggestion, $importer) {
$a = get_app(); $a = get_app();
logger("Processing suggestions"); logger("Processing suggestions");
@ -1556,7 +1565,7 @@ class dfrn {
* @param object $relocation relocation elements * @param object $relocation relocation elements
* @param array $importer Record of the importer user mixed with contact of the content * @param array $importer Record of the importer user mixed with contact of the content
*/ */
private function process_relocation($xpath, $relocation, $importer) { private static function process_relocation($xpath, $relocation, $importer) {
logger("Processing relocations"); logger("Processing relocations");
@ -1685,7 +1694,7 @@ class dfrn {
* @param array $importer Record of the importer user mixed with contact of the content * @param array $importer Record of the importer user mixed with contact of the content
* @param int $entrytype Is it a toplevel entry, a comment or a relayed comment? * @param int $entrytype Is it a toplevel entry, a comment or a relayed comment?
*/ */
private function update_content($current, $item, $importer, $entrytype) { private static function update_content($current, $item, $importer, $entrytype) {
$changed = false; $changed = false;
if (edited_timestamp_is_newer($current, $item)) { if (edited_timestamp_is_newer($current, $item)) {
@ -1737,7 +1746,7 @@ class dfrn {
* *
* @return int Is it a toplevel entry, a comment or a relayed comment? * @return int Is it a toplevel entry, a comment or a relayed comment?
*/ */
private function get_entry_type($importer, $item) { private static function get_entry_type($importer, $item) {
if ($item["parent-uri"] != $item["uri"]) { if ($item["parent-uri"] != $item["uri"]) {
$community = false; $community = false;
@ -1803,7 +1812,7 @@ class dfrn {
* @param array $importer Record of the importer user mixed with contact of the content * @param array $importer Record of the importer user mixed with contact of the content
* @param int $posted_id The record number of item record that was just posted * @param int $posted_id The record number of item record that was just posted
*/ */
private function do_poke($item, $importer, $posted_id) { private static function do_poke($item, $importer, $posted_id) {
$verb = urldecode(substr($item["verb"],strpos($item["verb"], "#")+1)); $verb = urldecode(substr($item["verb"],strpos($item["verb"], "#")+1));
if(!$verb) if(!$verb)
return; return;
@ -1858,7 +1867,7 @@ class dfrn {
* *
* @return bool Should the processing of the entries be continued? * @return bool Should the processing of the entries be continued?
*/ */
private function process_verbs($entrytype, $importer, &$item, &$is_like) { private static function process_verbs($entrytype, $importer, &$item, &$is_like) {
logger("Process verb ".$item["verb"]." and object-type ".$item["object-type"]." for entrytype ".$entrytype, LOGGER_DEBUG); logger("Process verb ".$item["verb"]." and object-type ".$item["object-type"]." for entrytype ".$entrytype, LOGGER_DEBUG);
@ -1958,7 +1967,7 @@ class dfrn {
* @param object $links link elements * @param object $links link elements
* @param array $item the item record * @param array $item the item record
*/ */
private function parse_links($links, &$item) { private static function parse_links($links, &$item) {
$rel = ""; $rel = "";
$href = ""; $href = "";
$type = ""; $type = "";
@ -2001,7 +2010,7 @@ class dfrn {
* @param object $entry entry elements * @param object $entry entry elements
* @param array $importer Record of the importer user mixed with contact of the content * @param array $importer Record of the importer user mixed with contact of the content
*/ */
private function process_entry($header, $xpath, $entry, $importer) { private static function process_entry($header, $xpath, $entry, $importer) {
logger("Processing entries"); logger("Processing entries");
@ -2010,6 +2019,20 @@ class dfrn {
// Get the uri // Get the uri
$item["uri"] = $xpath->query("atom:id/text()", $entry)->item(0)->nodeValue; $item["uri"] = $xpath->query("atom:id/text()", $entry)->item(0)->nodeValue;
$item["edited"] = $xpath->query("atom:updated/text()", $entry)->item(0)->nodeValue;
$current = q("SELECT `id`, `uid`, `last-child`, `edited`, `body` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
dbesc($item["uri"]),
intval($importer["importer_uid"])
);
// Is there an existing item?
if (dbm::is_result($current) AND edited_timestamp_is_newer($current[0], $item) AND
(datetime_convert("UTC","UTC",$item["edited"]) < $current[0]["edited"])) {
logger("Item ".$item["uri"]." already existed.", LOGGER_DEBUG);
return;
}
// Fetch the owner // Fetch the owner
$owner = self::fetchauthor($xpath, $entry, $importer, "dfrn:owner", true); $owner = self::fetchauthor($xpath, $entry, $importer, "dfrn:owner", true);
@ -2027,7 +2050,6 @@ class dfrn {
$item["title"] = $xpath->query("atom:title/text()", $entry)->item(0)->nodeValue; $item["title"] = $xpath->query("atom:title/text()", $entry)->item(0)->nodeValue;
$item["created"] = $xpath->query("atom:published/text()", $entry)->item(0)->nodeValue; $item["created"] = $xpath->query("atom:published/text()", $entry)->item(0)->nodeValue;
$item["edited"] = $xpath->query("atom:updated/text()", $entry)->item(0)->nodeValue;
$item["body"] = $xpath->query("dfrn:env/text()", $entry)->item(0)->nodeValue; $item["body"] = $xpath->query("dfrn:env/text()", $entry)->item(0)->nodeValue;
$item["body"] = str_replace(array(' ',"\t","\r","\n"), array('','','',''),$item["body"]); $item["body"] = str_replace(array(' ',"\t","\r","\n"), array('','','',''),$item["body"]);
@ -2215,18 +2237,13 @@ class dfrn {
} }
} }
$r = q("SELECT `id`, `uid`, `last-child`, `edited`, `body` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
dbesc($item["uri"]),
intval($importer["importer_uid"])
);
if (!self::process_verbs($entrytype, $importer, $item, $is_like)) { if (!self::process_verbs($entrytype, $importer, $item, $is_like)) {
logger("Exiting because 'process_verbs' told us so", LOGGER_DEBUG); logger("Exiting because 'process_verbs' told us so", LOGGER_DEBUG);
return; return;
} }
// Update content if 'updated' changes // Update content if 'updated' changes
if (dbm::is_result($r)) { if (dbm::is_result($current)) {
if (self::update_content($r[0], $item, $importer, $entrytype)) if (self::update_content($r[0], $item, $importer, $entrytype))
logger("Item ".$item["uri"]." was updated.", LOGGER_DEBUG); logger("Item ".$item["uri"]." was updated.", LOGGER_DEBUG);
else else
@ -2311,7 +2328,7 @@ class dfrn {
* @param object $deletion deletion elements * @param object $deletion deletion elements
* @param array $importer Record of the importer user mixed with contact of the content * @param array $importer Record of the importer user mixed with contact of the content
*/ */
private function process_deletion($xpath, $deletion, $importer) { private static function process_deletion($xpath, $deletion, $importer) {
logger("Processing deletions"); logger("Processing deletions");

42
include/feed.php
View file

@ -177,18 +177,6 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) {
foreach (array_reverse($entrylist) AS $entry) { foreach (array_reverse($entrylist) AS $entry) {
$item = array_merge($header, $author); $item = array_merge($header, $author);
$item["title"] = $xpath->evaluate('atom:title/text()', $entry)->item(0)->nodeValue;
if ($item["title"] == "")
$item["title"] = $xpath->evaluate('title/text()', $entry)->item(0)->nodeValue;
if ($item["title"] == "")
$item["title"] = $xpath->evaluate('rss:title/text()', $entry)->item(0)->nodeValue;
$alternate = $xpath->query("atom:link[@rel='alternate']", $entry)->item(0)->attributes;
if (!is_object($alternate))
$alternate = $xpath->query("atom:link", $entry)->item(0)->attributes;
if (is_object($alternate)) if (is_object($alternate))
foreach($alternate AS $attributes) foreach($alternate AS $attributes)
if ($attributes->name == "href") if ($attributes->name == "href")
@ -212,6 +200,27 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) {
$item["parent-uri"] = $item["uri"]; $item["parent-uri"] = $item["uri"];
if (!$simulate) {
$r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s', '%s')",
intval($importer["uid"]), dbesc($item["uri"]), dbesc(NETWORK_FEED), dbesc(NETWORK_DFRN));
if ($r) {
logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$r[0]["id"], LOGGER_DEBUG);
continue;
}
}
$item["title"] = $xpath->evaluate('atom:title/text()', $entry)->item(0)->nodeValue;
if ($item["title"] == "")
$item["title"] = $xpath->evaluate('title/text()', $entry)->item(0)->nodeValue;
if ($item["title"] == "")
$item["title"] = $xpath->evaluate('rss:title/text()', $entry)->item(0)->nodeValue;
$alternate = $xpath->query("atom:link[@rel='alternate']", $entry)->item(0)->attributes;
if (!is_object($alternate))
$alternate = $xpath->query("atom:link", $entry)->item(0)->attributes;
$published = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue; $published = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue;
if ($published == "") if ($published == "")
@ -250,15 +259,6 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) {
if ($creator != "") if ($creator != "")
$item["author-name"] = $creator; $item["author-name"] = $creator;
if (!$simulate) {
$r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s', '%s')",
intval($importer["uid"]), dbesc($item["uri"]), dbesc(NETWORK_FEED), dbesc(NETWORK_DFRN));
if ($r) {
logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$r[0]["id"], LOGGER_DEBUG);
continue;
}
}
/// @TODO ? /// @TODO ?
// <category>Ausland</category> // <category>Ausland</category>
// <media:thumbnail width="152" height="76" url="http://www.taz.de/picture/667875/192/14388767.jpg"/> // <media:thumbnail width="152" height="76" url="http://www.taz.de/picture/667875/192/14388767.jpg"/>

2
include/network.php
View file

@ -143,6 +143,8 @@ function z_fetch_url($url,$binary = false, &$redirects = 0, $opts=array()) {
logger('fetch_url error fetching '.$url.': '.curl_error($ch), LOGGER_NORMAL); logger('fetch_url error fetching '.$url.': '.curl_error($ch), LOGGER_NORMAL);
} }
$ret['errno'] = curl_errno($ch);
$base = $s; $base = $s;
$curl_info = @curl_getinfo($ch); $curl_info = @curl_getinfo($ch);

211
include/onepoll.php
View file

@ -15,11 +15,11 @@ function RemoveReply($subject) {
function onepoll_run(&$argv, &$argc){ function onepoll_run(&$argv, &$argc){
global $a, $db; global $a, $db;
if(is_null($a)) { if (is_null($a)) {
$a = new App; $a = new App;
} }
if(is_null($db)) { if (is_null($db)) {
@include(".htconfig.php"); @include(".htconfig.php");
require_once("include/dba.php"); require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data); $db = new dba($db_host, $db_user, $db_pass, $db_data);
@ -48,21 +48,25 @@ function onepoll_run(&$argv, &$argc){
$force = false; $force = false;
$restart = false; $restart = false;
if(($argc > 1) && (intval($argv[1]))) if (($argc > 1) && (intval($argv[1]))) {
$contact_id = intval($argv[1]); $contact_id = intval($argv[1]);
}
if(($argc > 2) && ($argv[2] == "force")) if (($argc > 2) && ($argv[2] == "force")) {
$force = true; $force = true;
}
if(! $contact_id) { if (! $contact_id) {
logger('onepoll: no contact'); logger('onepoll: no contact');
return; return;
} }
// Don't check this stuff if the function is called by the poller // Don't check this stuff if the function is called by the poller
if (App::callstack() != "poller_run") if (App::callstack() != "poller_run") {
if (App::is_already_running('onepoll'.$contact_id, '', 540)) if (App::is_already_running('onepoll'.$contact_id, '', 540)) {
return; return;
}
}
$d = datetime_convert(); $d = datetime_convert();
@ -83,8 +87,9 @@ function onepoll_run(&$argv, &$argc){
intval($contact_id) intval($contact_id)
); );
if(! count($contacts)) if (! count($contacts)) {
return; return;
}
$contact = $contacts[0]; $contact = $contacts[0];
@ -94,9 +99,11 @@ function onepoll_run(&$argv, &$argc){
where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY",
intval($contact['id']) intval($contact['id'])
); );
if (dbm::is_result($r)) if (dbm::is_result($r)) {
if (!$r[0]['total']) if (!$r[0]['total']) {
poco_load($contact['id'],$importer_uid,0,$contact['poco']); poco_load($contact['id'],$importer_uid,0,$contact['poco']);
}
}
} }
/// @TODO Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?) /// @TODO Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?)
@ -127,24 +134,24 @@ function onepoll_run(&$argv, &$argc){
$t = $contact['last-update']; $t = $contact['last-update'];
if($contact['subhub']) { if ($contact['subhub']) {
$poll_interval = get_config('system','pushpoll_frequency'); $poll_interval = get_config('system','pushpoll_frequency');
$contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3); $contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3);
$hub_update = false; $hub_update = false;
if(datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 day")) if (datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 day")) {
$hub_update = true; $hub_update = true;
} }
else } else {
$hub_update = false; $hub_update = false;
}
$importer_uid = $contact['uid']; $importer_uid = $contact['uid'];
$r = q("SELECT `contact`.*, `user`.`page-flags` FROM `contact` INNER JOIN `user` on `contact`.`uid` = `user`.`uid` WHERE `user`.`uid` = %d AND `contact`.`self` = 1 LIMIT 1", $r = q("SELECT `contact`.*, `user`.`page-flags` FROM `contact` INNER JOIN `user` on `contact`.`uid` = `user`.`uid` WHERE `user`.`uid` = %d AND `contact`.`self` = 1 LIMIT 1",
intval($importer_uid) intval($importer_uid)
); );
if (! dbm::is_result($r)) { if (!dbm::is_result($r)) {
return; return;
} }
@ -158,7 +165,7 @@ function onepoll_run(&$argv, &$argc){
); );
// Update the contact entry // Update the contact entry
if(($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) { if (($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) {
if (!poco_reachable($contact['url'])) { if (!poco_reachable($contact['url'])) {
logger("Skipping probably dead contact ".$contact['url']); logger("Skipping probably dead contact ".$contact['url']);
return; return;
@ -167,40 +174,50 @@ function onepoll_run(&$argv, &$argc){
if (!update_contact($contact["id"])) { if (!update_contact($contact["id"])) {
mark_for_death($contact); mark_for_death($contact);
return; return;
} else } else {
unmark_for_death($contact); unmark_for_death($contact);
}
} }
if($contact['network'] === NETWORK_DFRN) { if ($contact['network'] === NETWORK_DFRN) {
$idtosend = $orig_id = (($contact['dfrn-id']) ? $contact['dfrn-id'] : $contact['issued-id']); $idtosend = $orig_id = (($contact['dfrn-id']) ? $contact['dfrn-id'] : $contact['issued-id']);
if(intval($contact['duplex']) && $contact['dfrn-id']) if (intval($contact['duplex']) && $contact['dfrn-id']) {
$idtosend = '0:' . $orig_id; $idtosend = '0:' . $orig_id;
if(intval($contact['duplex']) && $contact['issued-id']) }
if (intval($contact['duplex']) && $contact['issued-id']) {
$idtosend = '1:' . $orig_id; $idtosend = '1:' . $orig_id;
}
// they have permission to write to us. We already filtered this in the contact query. // they have permission to write to us. We already filtered this in the contact query.
$perm = 'rw'; $perm = 'rw';
// But this may be our first communication, so set the writable flag if it isn't set already. // But this may be our first communication, so set the writable flag if it isn't set already.
if(! intval($contact['writable'])) if (! intval($contact['writable'])) {
q("update contact set writable = 1 where id = %d", intval($contact['id'])); q("update contact set writable = 1 where id = %d", intval($contact['id']));
}
$url = $contact['poll'] . '?dfrn_id=' . $idtosend $url = $contact['poll'] . '?dfrn_id=' . $idtosend
. '&dfrn_version=' . DFRN_PROTOCOL_VERSION . '&dfrn_version=' . DFRN_PROTOCOL_VERSION
. '&type=data&last_update=' . $last_update . '&type=data&last_update=' . $last_update
. '&perm=' . $perm ; . '&perm=' . $perm ;
$handshake_xml = fetch_url($url); $ret = z_fetch_url($url);
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return;
}
$handshake_xml = $ret['body'];
$html_code = $a->get_curl_code(); $html_code = $a->get_curl_code();
logger('onepoll: handshake with url ' . $url . ' returns xml: ' . $handshake_xml, LOGGER_DATA); logger('onepoll: handshake with url ' . $url . ' returns xml: ' . $handshake_xml, LOGGER_DATA);
if((! strlen($handshake_xml)) || ($html_code >= 400) || (! $html_code)) { if ((! strlen($handshake_xml)) || ($html_code >= 400) || (! $html_code)) {
logger("poller: $url appears to be dead - marking for death "); logger("poller: $url appears to be dead - marking for death ");
// dead connection - might be a transient event, or this might // dead connection - might be a transient event, or this might
@ -219,7 +236,7 @@ function onepoll_run(&$argv, &$argc){
return; return;
} }
if(! strstr($handshake_xml,'<')) { if (! strstr($handshake_xml,'<')) {
logger('poller: response from ' . $url . ' did not contain XML.'); logger('poller: response from ' . $url . ' did not contain XML.');
mark_for_death($contact); mark_for_death($contact);
@ -235,7 +252,7 @@ function onepoll_run(&$argv, &$argc){
$res = parse_xml_string($handshake_xml); $res = parse_xml_string($handshake_xml);
if(intval($res->status) == 1) { if (intval($res->status) == 1) {
logger("poller: $url replied status 1 - marking for death "); logger("poller: $url replied status 1 - marking for death ");
// we may not be friends anymore. Will keep trying for one month. // we may not be friends anymore. Will keep trying for one month.
@ -248,18 +265,16 @@ function onepoll_run(&$argv, &$argc){
intval($contact['id']) intval($contact['id'])
); );
mark_for_death($contact); mark_for_death($contact);
} } elseif ($contact['term-date'] != '0000-00-00 00:00:00') {
else { logger("poller: $url back from the dead - removing mark for death");
if($contact['term-date'] != '0000-00-00 00:00:00') { unmark_for_death($contact);
logger("poller: $url back from the dead - removing mark for death");
unmark_for_death($contact);
}
} }
if((intval($res->status) != 0) || (! strlen($res->challenge)) || (! strlen($res->dfrn_id))) if ((intval($res->status) != 0) || (! strlen($res->challenge)) || (! strlen($res->dfrn_id))) {
return; return;
}
if(((float) $res->dfrn_version > 2.21) && ($contact['poco'] == '')) { if (((float) $res->dfrn_version > 2.21) && ($contact['poco'] == '')) {
q("update contact set poco = '%s' where id = %d", q("update contact set poco = '%s' where id = %d",
dbesc(str_replace('/profile/','/poco/', $contact['url'])), dbesc(str_replace('/profile/','/poco/', $contact['url'])),
intval($contact['id']) intval($contact['id'])
@ -273,21 +288,21 @@ function onepoll_run(&$argv, &$argc){
$final_dfrn_id = ''; $final_dfrn_id = '';
if(($contact['duplex']) && strlen($contact['prvkey'])) { if (($contact['duplex']) && strlen($contact['prvkey'])) {
openssl_private_decrypt($sent_dfrn_id,$final_dfrn_id,$contact['prvkey']); openssl_private_decrypt($sent_dfrn_id,$final_dfrn_id,$contact['prvkey']);
openssl_private_decrypt($challenge,$postvars['challenge'],$contact['prvkey']); openssl_private_decrypt($challenge,$postvars['challenge'],$contact['prvkey']);
} } else {
else {
openssl_public_decrypt($sent_dfrn_id,$final_dfrn_id,$contact['pubkey']); openssl_public_decrypt($sent_dfrn_id,$final_dfrn_id,$contact['pubkey']);
openssl_public_decrypt($challenge,$postvars['challenge'],$contact['pubkey']); openssl_public_decrypt($challenge,$postvars['challenge'],$contact['pubkey']);
} }
$final_dfrn_id = substr($final_dfrn_id, 0, strpos($final_dfrn_id, '.')); $final_dfrn_id = substr($final_dfrn_id, 0, strpos($final_dfrn_id, '.'));
if(strpos($final_dfrn_id,':') == 1) if (strpos($final_dfrn_id,':') == 1) {
$final_dfrn_id = substr($final_dfrn_id,2); $final_dfrn_id = substr($final_dfrn_id,2);
}
if($final_dfrn_id != $orig_id) { if ($final_dfrn_id != $orig_id) {
logger('poller: ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id); logger('poller: ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id);
// did not decode properly - cannot trust this site // did not decode properly - cannot trust this site
return; return;
@ -299,10 +314,9 @@ function onepoll_run(&$argv, &$argc){
$xml = post_url($contact['poll'],$postvars); $xml = post_url($contact['poll'],$postvars);
} } elseif (($contact['network'] === NETWORK_OSTATUS)
elseif(($contact['network'] === NETWORK_OSTATUS)
|| ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DIASPORA)
|| ($contact['network'] === NETWORK_FEED) ) { || ($contact['network'] === NETWORK_FEED)) {
// Upgrading DB fields from an older Friendica version // Upgrading DB fields from an older Friendica version
// Will only do this once per notify-enabled OStatus contact // Will only do this once per notify-enabled OStatus contact
@ -311,10 +325,11 @@ function onepoll_run(&$argv, &$argc){
$stat_writeable = ((($contact['notify']) && ($contact['rel'] == CONTACT_IS_FOLLOWER || $contact['rel'] == CONTACT_IS_FRIEND)) ? 1 : 0); $stat_writeable = ((($contact['notify']) && ($contact['rel'] == CONTACT_IS_FOLLOWER || $contact['rel'] == CONTACT_IS_FRIEND)) ? 1 : 0);
// Contacts from OStatus are always writable // Contacts from OStatus are always writable
if($contact['network'] === NETWORK_OSTATUS) if ($contact['network'] === NETWORK_OSTATUS) {
$stat_writeable = 1; $stat_writeable = 1;
}
if($stat_writeable != $contact['writable']) { if ($stat_writeable != $contact['writable']) {
q("UPDATE `contact` SET `writable` = %d WHERE `id` = %d", q("UPDATE `contact` SET `writable` = %d WHERE `id` = %d",
intval($stat_writeable), intval($stat_writeable),
intval($contact['id']) intval($contact['id'])
@ -323,19 +338,26 @@ function onepoll_run(&$argv, &$argc){
// Are we allowed to import from this person? // Are we allowed to import from this person?
if($contact['rel'] == CONTACT_IS_FOLLOWER || $contact['blocked'] || $contact['readonly']) if ($contact['rel'] == CONTACT_IS_FOLLOWER || $contact['blocked'] || $contact['readonly']) {
return; return;
}
$cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-'); $cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-');
$xml = fetch_url($contact['poll'], false, $redirects, 0, Null, $cookiejar); $ret = z_fetch_url($contact['poll'], false, $redirects, array('cookiejar' => $cookiejar));
if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
return;
}
$xml = $ret['body'];
unlink($cookiejar); unlink($cookiejar);
} } elseif ($contact['network'] === NETWORK_MAIL || $contact['network'] === NETWORK_MAIL2) {
elseif($contact['network'] === NETWORK_MAIL || $contact['network'] === NETWORK_MAIL2) {
logger("Mail: Fetching", LOGGER_DEBUG); logger("Mail: Fetching", LOGGER_DEBUG);
$mail_disabled = ((function_exists('imap_open') && (! get_config('system','imap_disabled'))) ? 0 : 1); $mail_disabled = ((function_exists('imap_open') && (! get_config('system','imap_disabled'))) ? 0 : 1);
if($mail_disabled) if ($mail_disabled)
return; return;
logger("Mail: Enabled", LOGGER_DEBUG); logger("Mail: Enabled", LOGGER_DEBUG);
@ -347,38 +369,38 @@ function onepoll_run(&$argv, &$argc){
$mailconf = q("SELECT * FROM `mailacct` WHERE `server` != '' AND `uid` = %d LIMIT 1", $mailconf = q("SELECT * FROM `mailacct` WHERE `server` != '' AND `uid` = %d LIMIT 1",
intval($importer_uid) intval($importer_uid)
); );
if(count($x) && count($mailconf)) { if (count($x) && count($mailconf)) {
$mailbox = construct_mailbox_name($mailconf[0]); $mailbox = construct_mailbox_name($mailconf[0]);
$password = ''; $password = '';
openssl_private_decrypt(hex2bin($mailconf[0]['pass']),$password,$x[0]['prvkey']); openssl_private_decrypt(hex2bin($mailconf[0]['pass']),$password,$x[0]['prvkey']);
$mbox = email_connect($mailbox,$mailconf[0]['user'],$password); $mbox = email_connect($mailbox,$mailconf[0]['user'],$password);
unset($password); unset($password);
logger("Mail: Connect to " . $mailconf[0]['user']); logger("Mail: Connect to " . $mailconf[0]['user']);
if($mbox) { if ($mbox) {
q("UPDATE `mailacct` SET `last_check` = '%s' WHERE `id` = %d AND `uid` = %d", q("UPDATE `mailacct` SET `last_check` = '%s' WHERE `id` = %d AND `uid` = %d",
dbesc(datetime_convert()), dbesc(datetime_convert()),
intval($mailconf[0]['id']), intval($mailconf[0]['id']),
intval($importer_uid) intval($importer_uid)
); );
logger("Mail: Connected to " . $mailconf[0]['user']); logger("Mail: Connected to " . $mailconf[0]['user']);
} else } else {
logger("Mail: Connection error ".$mailconf[0]['user']." ".print_r(imap_errors(), true)); logger("Mail: Connection error ".$mailconf[0]['user']." ".print_r(imap_errors(), true));
}
} }
if($mbox) { if ($mbox) {
$msgs = email_poll($mbox,$contact['addr']); $msgs = email_poll($mbox,$contact['addr']);
if(count($msgs)) { if (count($msgs)) {
logger("Mail: Parsing ".count($msgs)." mails for ".$mailconf[0]['user'], LOGGER_DEBUG); logger("Mail: Parsing ".count($msgs)." mails for ".$mailconf[0]['user'], LOGGER_DEBUG);
$metas = email_msg_meta($mbox,implode(',',$msgs)); $metas = email_msg_meta($mbox,implode(',',$msgs));
if(count($metas) != count($msgs)) { if (count($metas) != count($msgs)) {
logger("onepoll: for " . $mailconf[0]['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", LOGGER_DEBUG); logger("onepoll: for " . $mailconf[0]['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", LOGGER_DEBUG);
} } else {
else {
$msgs = array_combine($msgs, $metas); $msgs = array_combine($msgs, $metas);
foreach($msgs as $msg_uid => $meta) { foreach ($msgs as $msg_uid => $meta) {
logger("Mail: Parsing mail ".$msg_uid, LOGGER_DATA); logger("Mail: Parsing mail ".$msg_uid, LOGGER_DATA);
$datarray = array(); $datarray = array();
@ -400,7 +422,7 @@ function onepoll_run(&$argv, &$argc){
// Only delete when mails aren't automatically moved or deleted // Only delete when mails aren't automatically moved or deleted
if (($mailconf[0]['action'] != 1) AND ($mailconf[0]['action'] != 3)) if (($mailconf[0]['action'] != 1) AND ($mailconf[0]['action'] != 3))
if($meta->deleted && ! $r[0]['deleted']) { if ($meta->deleted && ! $r[0]['deleted']) {
q("UPDATE `item` SET `deleted` = 1, `changed` = '%s' WHERE `id` = %d", q("UPDATE `item` SET `deleted` = 1, `changed` = '%s' WHERE `id` = %d",
dbesc(datetime_convert()), dbesc(datetime_convert()),
intval($r[0]['id']) intval($r[0]['id'])
@ -434,13 +456,13 @@ function onepoll_run(&$argv, &$argc){
// $raw_refs = ((x($headers,'references')) ? str_replace("\t",'',$headers['references']) : ''); // $raw_refs = ((x($headers,'references')) ? str_replace("\t",'',$headers['references']) : '');
$raw_refs = ((property_exists($meta,'references')) ? str_replace("\t",'',$meta->references) : ''); $raw_refs = ((property_exists($meta,'references')) ? str_replace("\t",'',$meta->references) : '');
if(! trim($raw_refs)) if (! trim($raw_refs))
$raw_refs = ((property_exists($meta,'in_reply_to')) ? str_replace("\t",'',$meta->in_reply_to) : ''); $raw_refs = ((property_exists($meta,'in_reply_to')) ? str_replace("\t",'',$meta->in_reply_to) : '');
$raw_refs = trim($raw_refs); // Don't allow a blank reference in $refs_arr $raw_refs = trim($raw_refs); // Don't allow a blank reference in $refs_arr
if($raw_refs) { if ($raw_refs) {
$refs_arr = explode(' ', $raw_refs); $refs_arr = explode(' ', $raw_refs);
if(count($refs_arr)) { if (count($refs_arr)) {
for($x = 0; $x < count($refs_arr); $x ++) for($x = 0; $x < count($refs_arr); $x ++)
$refs_arr[$x] = "'" . msgid2iri(str_replace(array('<','>',' '),array('','',''),dbesc($refs_arr[$x]))) . "'"; $refs_arr[$x] = "'" . msgid2iri(str_replace(array('<','>',' '),array('','',''),dbesc($refs_arr[$x]))) . "'";
} }
@ -456,12 +478,13 @@ function onepoll_run(&$argv, &$argc){
// Decoding the header // Decoding the header
$subject = imap_mime_header_decode($meta->subject); $subject = imap_mime_header_decode($meta->subject);
$datarray['title'] = ""; $datarray['title'] = "";
foreach($subject as $subpart) foreach ($subject as $subpart) {
if ($subpart->charset != "default") if ($subpart->charset != "default") {
$datarray['title'] .= iconv($subpart->charset, 'UTF-8//IGNORE', $subpart->text); $datarray['title'] .= iconv($subpart->charset, 'UTF-8//IGNORE', $subpart->text);
else } else {
$datarray['title'] .= $subpart->text; $datarray['title'] .= $subpart->text;
}
}
$datarray['title'] = notags(trim($datarray['title'])); $datarray['title'] = notags(trim($datarray['title']));
//$datarray['title'] = notags(trim($meta->subject)); //$datarray['title'] = notags(trim($meta->subject));
@ -476,7 +499,7 @@ function onepoll_run(&$argv, &$argc){
$datarray['title'] = RemoveReply($datarray['title']); $datarray['title'] = RemoveReply($datarray['title']);
// If it seems to be a reply but a header couldn't be found take the last message with matching subject // If it seems to be a reply but a header couldn't be found take the last message with matching subject
if(!x($datarray,'parent-uri') and $reply) { if (!x($datarray,'parent-uri') and $reply) {
$r = q("SELECT `uri` , `parent-uri` FROM `item` WHERE `title` = \"%s\" AND `uid` = %d AND `network` = '%s' ORDER BY `created` DESC LIMIT 1", $r = q("SELECT `uri` , `parent-uri` FROM `item` WHERE `title` = \"%s\" AND `uid` = %d AND `network` = '%s' ORDER BY `created` DESC LIMIT 1",
dbesc(protect_sprintf($datarray['title'])), dbesc(protect_sprintf($datarray['title'])),
intval($importer_uid), intval($importer_uid),
@ -485,12 +508,12 @@ function onepoll_run(&$argv, &$argc){
$datarray['parent-uri'] = $r[0]['parent-uri']; $datarray['parent-uri'] = $r[0]['parent-uri'];
} }
if(! x($datarray,'parent-uri')) if (! x($datarray,'parent-uri'))
$datarray['parent-uri'] = $datarray['uri']; $datarray['parent-uri'] = $datarray['uri'];
$r = email_get_msg($mbox,$msg_uid, $reply); $r = email_get_msg($mbox,$msg_uid, $reply);
if(! $r) { if (! $r) {
logger("Mail: can't fetch msg ".$msg_uid." for ".$mailconf[0]['user']); logger("Mail: can't fetch msg ".$msg_uid." for ".$mailconf[0]['user']);
continue; continue;
} }
@ -502,23 +525,26 @@ function onepoll_run(&$argv, &$argc){
// some mailing lists have the original author as 'from' - add this sender info to msg body. // some mailing lists have the original author as 'from' - add this sender info to msg body.
/// @TODO Adding a gravatar for the original author would be cool /// @TODO Adding a gravatar for the original author would be cool
if(! stristr($meta->from,$contact['addr'])) { if (! stristr($meta->from,$contact['addr'])) {
$from = imap_mime_header_decode($meta->from); $from = imap_mime_header_decode($meta->from);
$fromdecoded = ""; $fromdecoded = "";
foreach($from as $frompart) foreach ($from as $frompart) {
if ($frompart->charset != "default") if ($frompart->charset != "default") {
$fromdecoded .= iconv($frompart->charset, 'UTF-8//IGNORE', $frompart->text); $fromdecoded .= iconv($frompart->charset, 'UTF-8//IGNORE', $frompart->text);
else } else {
$fromdecoded .= $frompart->text; $fromdecoded .= $frompart->text;
}
}
$fromarr = imap_rfc822_parse_adrlist($fromdecoded, $a->get_hostname()); $fromarr = imap_rfc822_parse_adrlist($fromdecoded, $a->get_hostname());
$frommail = $fromarr[0]->mailbox."@".$fromarr[0]->host; $frommail = $fromarr[0]->mailbox."@".$fromarr[0]->host;
if (isset($fromarr[0]->personal)) if (isset($fromarr[0]->personal)) {
$fromname = $fromarr[0]->personal; $fromname = $fromarr[0]->personal;
else } else {
$fromname = $frommail; $fromname = $frommail;
}
//$datarray['body'] = "[b]".t('From: ') . escape_tags($fromdecoded) . "[/b]\n\n" . $datarray['body']; //$datarray['body'] = "[b]".t('From: ') . escape_tags($fromdecoded) . "[/b]\n\n" . $datarray['body'];
@ -538,9 +564,9 @@ function onepoll_run(&$argv, &$argc){
$datarray['uid'] = $importer_uid; $datarray['uid'] = $importer_uid;
$datarray['contact-id'] = $contact['id']; $datarray['contact-id'] = $contact['id'];
if($datarray['parent-uri'] === $datarray['uri']) if ($datarray['parent-uri'] === $datarray['uri'])
$datarray['private'] = 1; $datarray['private'] = 1;
if(($contact['network'] === NETWORK_MAIL) && (! get_pconfig($importer_uid,'system','allow_public_email_replies'))) { if (($contact['network'] === NETWORK_MAIL) && (! get_pconfig($importer_uid,'system','allow_public_email_replies'))) {
$datarray['private'] = 1; $datarray['private'] = 1;
$datarray['allow_cid'] = '<' . $contact['id'] . '>'; $datarray['allow_cid'] = '<' . $contact['id'] . '>';
} }
@ -574,24 +600,24 @@ function onepoll_run(&$argv, &$argc){
} }
} }
} }
} else } else {
logger("Mail: no mails for ".$mailconf[0]['user']); logger("Mail: no mails for ".$mailconf[0]['user']);
}
logger("Mail: closing connection for ".$mailconf[0]['user']); logger("Mail: closing connection for ".$mailconf[0]['user']);
imap_close($mbox); imap_close($mbox);
} }
} } elseif ($contact['network'] === NETWORK_FACEBOOK) {
elseif($contact['network'] === NETWORK_FACEBOOK) {
// This is picked up by the Facebook plugin on a cron hook. // This is picked up by the Facebook plugin on a cron hook.
// Ignored here. // Ignored here.
} elseif($contact['network'] === NETWORK_PUMPIO) { } elseif ($contact['network'] === NETWORK_PUMPIO) {
// This is picked up by the pump.io plugin on a cron hook. // This is picked up by the pump.io plugin on a cron hook.
// Ignored here. // Ignored here.
} }
if($xml) { if ($xml) {
logger('poller: received xml : ' . $xml, LOGGER_DATA); logger('poller: received xml : ' . $xml, LOGGER_DATA);
if(! strstr($xml,'<')) { if (! strstr($xml,'<')) {
logger('poller: post_handshake: response from ' . $url . ' did not contain XML.'); logger('poller: post_handshake: response from ' . $url . ' did not contain XML.');
$r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d",
dbesc(datetime_convert()), dbesc(datetime_convert()),
@ -611,10 +637,10 @@ function onepoll_run(&$argv, &$argc){
consume_feed($xml,$importer,$contact,$hub,1,2); consume_feed($xml,$importer,$contact,$hub,1,2);
$hubmode = 'subscribe'; $hubmode = 'subscribe';
if($contact['network'] === NETWORK_DFRN || $contact['blocked'] || $contact['readonly']) if ($contact['network'] === NETWORK_DFRN || $contact['blocked'] || $contact['readonly'])
$hubmode = 'unsubscribe'; $hubmode = 'unsubscribe';
if(($contact['network'] === NETWORK_OSTATUS || $contact['network'] == NETWORK_FEED) && (! $contact['hub-verify'])) if (($contact['network'] === NETWORK_OSTATUS || $contact['network'] == NETWORK_FEED) && (! $contact['hub-verify']))
$hub_update = true; $hub_update = true;
if ($force) if ($force)
@ -622,14 +648,15 @@ function onepoll_run(&$argv, &$argc){
logger("Contact ".$contact['id']." returned hub: ".$hub." Network: ".$contact['network']." Relation: ".$contact['rel']." Update: ".$hub_update); logger("Contact ".$contact['id']." returned hub: ".$hub." Network: ".$contact['network']." Relation: ".$contact['rel']." Update: ".$hub_update);
if((strlen($hub)) && ($hub_update) && (($contact['rel'] != CONTACT_IS_FOLLOWER) || $contact['network'] == NETWORK_FEED) ) { if ((strlen($hub)) && ($hub_update) && (($contact['rel'] != CONTACT_IS_FOLLOWER) || $contact['network'] == NETWORK_FEED) ) {
logger('poller: hub ' . $hubmode . ' : ' . $hub . ' contact name : ' . $contact['name'] . ' local user : ' . $importer['name']); logger('poller: hub ' . $hubmode . ' : ' . $hub . ' contact name : ' . $contact['name'] . ' local user : ' . $importer['name']);
$hubs = explode(',', $hub); $hubs = explode(',', $hub);
if(count($hubs)) { if (count($hubs)) {
foreach($hubs as $h) { foreach ($hubs as $h) {
$h = trim($h); $h = trim($h);
if(! strlen($h)) if (! strlen($h)) {
continue; continue;
}
subscribe_to_hub($h,$importer,$contact,$hubmode); subscribe_to_hub($h,$importer,$contact,$hubmode);
} }
} }
@ -671,7 +698,7 @@ function onepoll_run(&$argv, &$argc){
return; return;
} }
if (array_search(__file__,get_included_files())===0){ if (array_search(__file__,get_included_files())===0) {
onepoll_run($_SERVER["argv"],$_SERVER["argc"]); onepoll_run($_SERVER["argv"],$_SERVER["argc"]);
killme(); killme();
} }

10
include/poller.php
View file

@ -207,6 +207,16 @@ function poller_exec_function($queue, $funcname, $argv) {
$duration = number_format(microtime(true) - $stamp, 3); $duration = number_format(microtime(true) - $stamp, 3);
if ($duration > 3600) {
logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 1 hour (".round($duration/60, 3).")", LOGGER_DEBUG);
} elseif ($duration > 600) {
logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 10 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
} elseif ($duration > 300) {
logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 5 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
} elseif ($duration > 120) {
logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
}
logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds.");
// Write down the performance values into the log // Write down the performance values into the log

89
include/queue.php
View file

@ -5,15 +5,16 @@ use \Friendica\Core\Config;
require_once("boot.php"); require_once("boot.php");
require_once('include/queue_fn.php'); require_once('include/queue_fn.php');
require_once('include/dfrn.php'); require_once('include/dfrn.php');
require_once('include/cache.php');
function queue_run(&$argv, &$argc){ function queue_run(&$argv, &$argc){
global $a, $db; global $a, $db;
if(is_null($a)){ if (is_null($a)){
$a = new App; $a = new App;
} }
if(is_null($db)){ if (is_null($db)){
@include(".htconfig.php"); @include(".htconfig.php");
require_once("include/dba.php"); require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data); $db = new dba($db_host, $db_user, $db_pass, $db_data);
@ -37,14 +38,14 @@ function queue_run(&$argv, &$argc){
load_hooks(); load_hooks();
if($argc > 1) if ($argc > 1) {
$queue_id = intval($argv[1]); $queue_id = intval($argv[1]);
else } else {
$queue_id = 0; $queue_id = 0;
}
$deadguys = array(); $cachekey_deadguy = 'queue_run:deadguy:';
$deadservers = array(); $cachekey_server = 'queue_run:server:';
$serverlist = array();
if (!$queue_id) { if (!$queue_id) {
@ -64,7 +65,7 @@ function queue_run(&$argv, &$argc){
foreach ($r as $rr) { foreach ($r as $rr) {
logger('queue: deliverq'); logger('queue: deliverq');
proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']); proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']);
if($interval) { if ($interval) {
time_sleep_until(microtime(true) + (float) $interval); time_sleep_until(microtime(true) + (float) $interval);
} }
} }
@ -111,10 +112,10 @@ function queue_run(&$argv, &$argc){
// queue_predeliver hooks may have changed the queue db details, // queue_predeliver hooks may have changed the queue db details,
// so check again if this entry still needs processing // so check again if this entry still needs processing
if($queue_id) if ($queue_id) {
$qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1", $qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id)); intval($queue_id));
elseif (get_config("system", "worker")) { } elseif (get_config("system", "worker")) {
logger('Call queue for id '.$q_item['id']); logger('Call queue for id '.$q_item['id']);
proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']); proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']);
continue; continue;
@ -122,8 +123,9 @@ function queue_run(&$argv, &$argc){
$qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ", $qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ",
intval($q_item['id'])); intval($q_item['id']));
if(! count($qi)) if (!dbm::is_result($qi)) {
continue; continue;
}
$c = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1", $c = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1",
@ -133,26 +135,32 @@ function queue_run(&$argv, &$argc){
remove_queue_item($q_item['id']); remove_queue_item($q_item['id']);
continue; continue;
} }
if(in_array($c[0]['notify'],$deadguys)) {
logger('queue: skipping known dead url: ' . $c[0]['notify']); $dead = Cache::get($cachekey_deadguy.$c[0]['notify']);
if (!is_null($dead) AND $dead) {
logger('queue: skipping known dead url: '.$c[0]['notify']);
update_queue_time($q_item['id']); update_queue_time($q_item['id']);
continue; continue;
} }
$server = poco_detect_server($c[0]['url']); $server = poco_detect_server($c[0]['url']);
if (($server != "") AND !in_array($server, $serverlist)) { if ($server != "") {
logger("Check server ".$server." (".$c[0]["network"].")"); $vital = Cache::get($cachekey_server.$server);
if (!poco_check_server($server, $c[0]["network"], true))
$deadservers[] = $server;
$serverlist[] = $server; if (is_null($vital)) {
} logger("Check server ".$server." (".$c[0]["network"].")");
if (($server != "") AND in_array($server, $deadservers)) { $vital = poco_check_server($server, $c[0]["network"], true);
logger('queue: skipping known dead server: '.$server); Cache::set($cachekey_server.$server, $vital, CACHE_QUARTER_HOUR);
update_queue_time($q_item['id']); }
continue;
if (!is_null($vital) AND !$vital) {
logger('queue: skipping dead server: '.$server);
update_queue_time($q_item['id']);
continue;
}
} }
$u = q("SELECT `user`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey` $u = q("SELECT `user`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey`
@ -176,36 +184,37 @@ function queue_run(&$argv, &$argc){
logger('queue: dfrndelivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>'); logger('queue: dfrndelivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = dfrn::deliver($owner,$contact,$data); $deliver_status = dfrn::deliver($owner,$contact,$data);
if($deliver_status == (-1)) { if ($deliver_status < 0) {
update_queue_time($q_item['id']); update_queue_time($q_item['id']);
$deadguys[] = $contact['notify']; Cache::set($cachekey_deadguy.$contact['notify'], true, CACHE_QUARTER_HOUR);
} else } else {
remove_queue_item($q_item['id']); remove_queue_item($q_item['id']);
}
break; break;
case NETWORK_OSTATUS: case NETWORK_OSTATUS:
if($contact['notify']) { if ($contact['notify']) {
logger('queue: slapdelivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>'); logger('queue: slapdelivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = slapper($owner,$contact['notify'],$data); $deliver_status = slapper($owner,$contact['notify'],$data);
if($deliver_status == (-1)) { if ($deliver_status == (-1)) {
update_queue_time($q_item['id']); update_queue_time($q_item['id']);
$deadguys[] = $contact['notify']; Cache::set($cachekey_deadguy.$contact['notify'], true, CACHE_QUARTER_HOUR);
} else } else {
remove_queue_item($q_item['id']); remove_queue_item($q_item['id']);
}
} }
break; break;
case NETWORK_DIASPORA: case NETWORK_DIASPORA:
if($contact['notify']) { if ($contact['notify']) {
logger('queue: diaspora_delivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>'); logger('queue: diaspora_delivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = Diaspora::transmit($owner,$contact,$data,$public,true); $deliver_status = Diaspora::transmit($owner,$contact,$data,$public,true);
if($deliver_status == (-1)) { if ($deliver_status == (-1)) {
update_queue_time($q_item['id']); update_queue_time($q_item['id']);
$deadguys[] = $contact['notify']; Cache::set($cachekey_deadguy.$contact['notify'], true, CACHE_QUARTER_HOUR);
} else } else {
remove_queue_item($q_item['id']); remove_queue_item($q_item['id']);
}
} }
break; break;
@ -213,15 +222,15 @@ function queue_run(&$argv, &$argc){
$params = array('owner' => $owner, 'contact' => $contact, 'queue' => $q_item, 'result' => false); $params = array('owner' => $owner, 'contact' => $contact, 'queue' => $q_item, 'result' => false);
call_hooks('queue_deliver', $a, $params); call_hooks('queue_deliver', $a, $params);
if($params['result']) if ($params['result']) {
remove_queue_item($q_item['id']); remove_queue_item($q_item['id']);
else } else {
update_queue_time($q_item['id']); update_queue_time($q_item['id']);
}
break; break;
} }
logger('Deliver status '.$deliver_status.' for item '.$q_item['id'].' to '.$contact['name'].' <'.$contact['url'].'>'); logger('Deliver status '.(int)$deliver_status.' for item '.$q_item['id'].' to '.$contact['name'].' <'.$contact['url'].'>');
} }
return; return;

27
include/socgraph.php
View file

@ -690,7 +690,7 @@ function poco_check_server($server_url, $network = "", $force = false) {
return false; return false;
$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); $servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url)));
if ($servers) { if (dbm::is_result($servers)) {
if ($servers[0]["created"] == "0000-00-00 00:00:00") if ($servers[0]["created"] == "0000-00-00 00:00:00")
q("UPDATE `gserver` SET `created` = '%s' WHERE `nurl` = '%s'", q("UPDATE `gserver` SET `created` = '%s' WHERE `nurl` = '%s'",
@ -732,21 +732,40 @@ function poco_check_server($server_url, $network = "", $force = false) {
$orig_last_failure = $last_failure; $orig_last_failure = $last_failure;
// Check if the page is accessible via SSL. // Check if the page is accessible via SSL.
$orig_server_url = $server_url;
$server_url = str_replace("http://", "https://", $server_url); $server_url = str_replace("http://", "https://", $server_url);
$serverret = z_fetch_url($server_url."/.well-known/host-meta");
// We set the timeout to 20 seconds since this operation should be done in no time if the server was vital
$serverret = z_fetch_url($server_url."/.well-known/host-meta", false, $redirects, array('timeout' => 20));
// Quit if there is a timeout.
// But we want to make sure to only quit if we are mostly sure that this server url fits.
if (dbm::is_result($servers) AND ($orig_server_url == $server_url) AND
($serverret['errno'] == CURLE_OPERATION_TIMEDOUT)) {
logger("Connection to server ".$server_url." timed out.", LOGGER_DEBUG);
return false;
}
// Maybe the page is unencrypted only? // Maybe the page is unencrypted only?
$xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0"); $xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0");
if (!$serverret["success"] OR ($serverret["body"] == "") OR (@sizeof($xmlobj) == 0) OR !is_object($xmlobj)) { if (!$serverret["success"] OR ($serverret["body"] == "") OR (@sizeof($xmlobj) == 0) OR !is_object($xmlobj)) {
$server_url = str_replace("https://", "http://", $server_url); $server_url = str_replace("https://", "http://", $server_url);
$serverret = z_fetch_url($server_url."/.well-known/host-meta");
// We set the timeout to 20 seconds since this operation should be done in no time if the server was vital
$serverret = z_fetch_url($server_url."/.well-known/host-meta", false, $redirects, array('timeout' => 20));
// Quit if there is a timeout
if ($serverret['errno'] == CURLE_OPERATION_TIMEDOUT) {
logger("Connection to server ".$server_url." timed out.", LOGGER_DEBUG);
return false;
}
$xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0"); $xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0");
} }
if (!$serverret["success"] OR ($serverret["body"] == "") OR (sizeof($xmlobj) == 0) OR !is_object($xmlobj)) { if (!$serverret["success"] OR ($serverret["body"] == "") OR (sizeof($xmlobj) == 0) OR !is_object($xmlobj)) {
// Workaround for bad configured servers (known nginx problem) // Workaround for bad configured servers (known nginx problem)
if ($serverret["debug"]["http_code"] != "403") { if (!in_array($serverret["debug"]["http_code"], array("403", "404"))) {
$last_failure = datetime_convert(); $last_failure = datetime_convert();
$failure = true; $failure = true;
} }

7
mod/nodeinfo.php
View file

@ -228,9 +228,10 @@ function nodeinfo_cron() {
logger("local_posts: ".$local_posts, LOGGER_DEBUG); logger("local_posts: ".$local_posts, LOGGER_DEBUG);
$posts = qu("SELECT COUNT(*) AS `local_comments` FROM `item` $posts = qu("SELECT COUNT(*) FROM `contact`
INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` INNER JOIN `item` ON `item`.`contact-id` = `contact`.`id` AND `item`.`uid` = `contact`.`uid` AND
WHERE `contact`.`self` and `item`.`id` != `item`.`parent` AND `item`.`network` IN ('%s', '%s', '%s')", `item`.`id` != `item`.`parent` AND `item`.`network` IN ('%s', '%s', '%s')
WHERE `contact`.`self`",
dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_DFRN)); dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_DFRN));
if (!is_array($posts)) if (!is_array($posts))