From 4b403e4fe2e38b4387dbf275dd069a5c415cab5b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 10 Sep 2017 07:21:23 +0000 Subject: [PATCH 1/9] OStatus: Reworked import --- include/cron.php | 6 - include/cronjobs.php | 14 - include/dfrn.php | 30 +- include/ostatus.php | 1623 +++++++++++++----------------------------- mod/display.php | 2 +- 5 files changed, 532 insertions(+), 1143 deletions(-) diff --git a/include/cron.php b/include/cron.php index ccac49b63..8b0fab77a 100644 --- a/include/cron.php +++ b/include/cron.php @@ -42,12 +42,6 @@ function cron_run(&$argv, &$argc){ // Expire and remove user entries proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "expire_and_remove_users"); - // Check OStatus conversations - proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "ostatus_mentions"); - - // Check every conversation - proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "ostatus_conversations"); - // Call possible post update functions proc_run(PRIORITY_LOW, "include/cronjobs.php", "post_update"); diff --git a/include/cronjobs.php b/include/cronjobs.php index 53316e464..437fedd53 100644 --- a/include/cronjobs.php +++ b/include/cronjobs.php @@ -8,7 +8,6 @@ function cronjobs_run(&$argv, &$argc){ global $a; require_once 'include/datetime.php'; - require_once 'include/ostatus.php'; require_once 'include/post_update.php'; require_once 'mod/nodeinfo.php'; require_once 'include/photos.php'; @@ -22,19 +21,6 @@ function cronjobs_run(&$argv, &$argc){ logger("Starting cronjob ".$argv[1], LOGGER_DEBUG); - // Check OStatus conversations - // Check only conversations with mentions (for a longer time) - if ($argv[1] == 'ostatus_mentions') { - ostatus::check_conversations(true); - return; - } - - // Check every conversation - if ($argv[1] == 'ostatus_conversations') { - ostatus::check_conversations(false); - return; - } - // Call possible post update functions // see include/post_update.php for more details if ($argv[1] == 'post_update') { diff --git a/include/dfrn.php b/include/dfrn.php index 4c8f828af..0b12f4d61 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -303,7 +303,7 @@ class dfrn { AND (NOT `contact`.`blocked` OR `contact`.`pending`) LEFT JOIN `sign` ON `sign`.`iid` = `item`.`id` WHERE `item`.`id` = %d AND `item`.`visible` AND NOT `item`.`moderated` AND `item`.`parent` != 0 - AND `item`.`wall` AND NOT `item`.`private`", + AND NOT `item`.`private`", intval($item_id) ); @@ -330,12 +330,9 @@ class dfrn { $alternatelink = $owner['url']; - $root = self::add_header($doc, $owner, 'dfrn:owner', $alternatelink, true); - $type = 'html'; - $entry = self::entry($doc, $type, $item, $owner, true); - $root->appendChild($entry); + $root = self::entry($doc, $type, $item, $owner, true, 0, true); $atom = trim($doc->saveXML()); return $atom; @@ -827,7 +824,7 @@ class dfrn { * @return object XML entry object * @todo Add type-hints */ - private static function entry($doc, $type, $item, $owner, $comment = false, $cid = 0) { + private static function entry($doc, $type, $item, $owner, $comment = false, $cid = 0, $single = false) { $mentioned = array(); @@ -840,7 +837,22 @@ class dfrn { return xml::create_element($doc, "at:deleted-entry", "", $attributes); } - $entry = $doc->createElement("entry"); + if (!$single) { + $entry = $doc->createElement("entry"); + } else { + $entry = $doc->createElementNS(NAMESPACE_ATOM1, 'entry'); + $doc->appendChild($entry); + + $entry->setAttribute("xmlns:thr", NAMESPACE_THREAD); + $entry->setAttribute("xmlns:at", NAMESPACE_TOMB); + $entry->setAttribute("xmlns:media", NAMESPACE_MEDIA); + $entry->setAttribute("xmlns:dfrn", NAMESPACE_DFRN); + $entry->setAttribute("xmlns:activity", NAMESPACE_ACTIVITY); + $entry->setAttribute("xmlns:georss", NAMESPACE_GEORSS); + $entry->setAttribute("xmlns:poco", NAMESPACE_POCO); + $entry->setAttribute("xmlns:ostatus", NAMESPACE_OSTATUS); + $entry->setAttribute("xmlns:statusnet", NAMESPACE_STATUSNET); + } if ($item['allow_cid'] || $item['allow_gid'] || $item['deny_cid'] || $item['deny_gid']) { $body = fix_private_photos($item['body'],$owner['uid'],$item,$cid); @@ -868,10 +880,10 @@ class dfrn { $entry->appendChild($dfrnowner); if (($item['parent'] != $item['id']) || ($item['parent-uri'] !== $item['uri']) || (($item['thr-parent'] !== '') && ($item['thr-parent'] !== $item['uri']))) { - $parent = q("SELECT `guid` FROM `item` WHERE `id` = %d", intval($item["parent"])); $parent_item = (($item['thr-parent']) ? $item['thr-parent'] : $item['parent-uri']); + $parent = q("SELECT `guid`,`plink` FROM `item` WHERE `uri` = '%s' AND `uid` = %d", dbesc($parent_item), intval($item['uid'])); $attributes = array("ref" => $parent_item, "type" => "text/html", - "href" => System::baseUrl().'/display/'.$parent[0]['guid'], + "href" => $parent[0]['plink'], "dfrn:diaspora_guid" => $parent[0]['guid']); xml::add_element($doc, $entry, "thr:in-reply-to", "", $attributes); } diff --git a/include/ostatus.php b/include/ostatus.php index 1f81d9615..124f421bf 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -33,6 +33,516 @@ class ostatus { const OSTATUS_DEFAULT_POLL_TIMEFRAME = 1440; // given in minutes const OSTATUS_DEFAULT_POLL_TIMEFRAME_MENTIONS = 14400; // given in minutes + private static $itemlist; + + /** + * @brief Imports an XML string containing OStatus elements + * + * @param string $xml The XML + * @param array $importer user record of the importing user + * @param $contact + * @param array $hub Called by reference, returns the fetched hub data + */ + public static function import($xml, $importer, &$contact, &$hub) { + self::process($xml, $importer, $contact, $hub); + } + + /** + * @brief Imports an XML string containing OStatus elements + * + * @param string $xml The XML + * @param array $importer user record of the importing user + * @param $contact + * @param array $hub Called by reference, returns the fetched hub data + */ + private static function process($xml, $importer, &$contact, &$hub, $stored = false, $initialize = true) { + if ($initialize) { + self::$itemlist = array(); + } + + logger("Import OStatus message", LOGGER_DEBUG); + + if ($xml == "") { + return false; + } + $doc = new DOMDocument(); + @$doc->loadXML($xml); + + $xpath = new DomXPath($doc); + $xpath->registerNamespace('atom', NAMESPACE_ATOM1); + $xpath->registerNamespace('thr', NAMESPACE_THREAD); + $xpath->registerNamespace('georss', NAMESPACE_GEORSS); + $xpath->registerNamespace('activity', NAMESPACE_ACTIVITY); + $xpath->registerNamespace('media', NAMESPACE_MEDIA); + $xpath->registerNamespace('poco', NAMESPACE_POCO); + $xpath->registerNamespace('ostatus', NAMESPACE_OSTATUS); + $xpath->registerNamespace('statusnet', NAMESPACE_STATUSNET); + + $hub = ""; + $hub_attributes = $xpath->query("/atom:feed/atom:link[@rel='hub']")->item(0)->attributes; + if (is_object($hub_attributes)) { + foreach ($hub_attributes AS $hub_attribute) { + if ($hub_attribute->name == "href") { + $hub = $hub_attribute->textContent; + logger("Found hub ".$hub, LOGGER_DEBUG); + } + } + } + + $header = array(); + $header["uid"] = $importer["uid"]; + $header["network"] = NETWORK_OSTATUS; + $header["type"] = "remote"; + $header["wall"] = 0; + $header["origin"] = 0; + $header["gravity"] = GRAVITY_PARENT; + + $first_child = $doc->firstChild->tagName; + + if ($first_child == "feed") { + $entries = $xpath->query('/atom:feed/atom:entry'); + $header["protocol"] = PROTOCOL_OSTATUS_FEED; + } else { + $entries = $xpath->query('/atom:entry'); + $header["protocol"] = PROTOCOL_OSTATUS_SALMON; + } + + // Fetch the first author + $authordata = $xpath->query('//author')->item(0); + $author = self::fetchauthor($xpath, $authordata, $importer, $contact, $stored); + + $entry = $xpath->query('/atom:entry'); + $header["protocol"] = PROTOCOL_OSTATUS_SALMON; + + // Reverse the order of the entries + $entrylist = array(); + + foreach ($entries AS $entry) { + $entrylist[] = $entry; + } + + if (!$initialize && (count($entrylist) > 1)) { + return false; + } + + foreach (array_reverse($entrylist) AS $entry) { + // fetch the author + $authorelement = $xpath->query('/atom:entry/atom:author', $entry); + if ($authorelement->length > 0) { + $author = self::fetchauthor($xpath, $entry, $importer, $contact, $stored); + } + + $value = $xpath->evaluate('atom:author/poco:preferredUsername/text()', $entry)->item(0)->nodeValue; + if ($value != "") { + $nickname = $value; + } else { + $nickname = $author["author-name"]; + } + + $item = array_merge($header, $author); + + $item["verb"] = $xpath->query('activity:verb/text()', $entry)->item(0)->nodeValue; + + /// Delete a message + if ($item["verb"] == "qvitter-delete-notice" || $item["verb"] == ACTIVITY_DELETE) { + // ignore "Delete" messages (by now) + logger("Ignore delete message ".print_r($item, true)); + continue; + } + + if ($item["verb"] == ACTIVITY_JOIN) { + // ignore "Join" messages + logger("Ignore join message ".print_r($item, true)); + continue; + } + + if ($item["verb"] == ACTIVITY_FOLLOW) { + new_follower($importer, $contact, $item, $nickname); + continue; + } + + if ($item["verb"] == NAMESPACE_OSTATUS."/unfollow") { + lose_follower($importer, $contact, $item, $dummy); + continue; + } + + if ($item["verb"] == NAMESPACE_OSTATUS."/unfavorite") { + // Ignore "Unfavorite" message + logger("Ignore unfavorite message ".print_r($item, true)); + continue; + } + + if ($item["verb"] == ACTIVITY_FAVORITE) { + $orig_uri = $xpath->query("activity:object/atom:id", $entry)->item(0)->nodeValue; + logger("Favorite ".$orig_uri." ".print_r($item, true)); + + $item["verb"] = ACTIVITY_LIKE; + $item["parent-uri"] = $orig_uri; + $item["gravity"] = GRAVITY_LIKE; + } + + // http://activitystrea.ms/schema/1.0/rsvp-yes + if (!in_array($item["verb"], array(ACTIVITY_POST, ACTIVITY_LIKE, ACTIVITY_SHARE))) { + logger("Unhandled verb ".$item["verb"]." ".print_r($item, true)); + } + + $doc2 = new DOMDocument(); + $doc2->loadXML($xml); + $doc2->preserveWhiteSpace = false; + $doc2->formatOutput = true; + $xml2 = $doc2->saveXML(); + + $item["source"] = $xml2; + + self::processPost($xpath, $entry, $item, $importer); + + if ($initialize && (count(self::$itemlist) > 0)) { + // We will import it everytime, when it is started by our contacts + $valid = !empty(self::$itemlist[0]['contact-id']); + if (!$valid) { + // If not, then it depends on this setting + $valid = !Config::get('system','ostatus_full_threads'); + } + + if ($valid) { + // But we will only import complete threads + $valid = self::$itemlist[0]['uri'] == self::$itemlist[0]['parent-uri']; + } + + if ($valid) { + // Never post a thread when the only interaction by our contact was a like + $valid = false; + $verbs = array(ACTIVITY_POST, ACTIVITY_SHARE); + foreach (self::$itemlist AS $item) { + if (!empty($item['contact-id']) && in_array($item['verb'], $verbs)) { + $valid = true; + } + } + } + + if ($valid) { + $default_contact = 0; + $key = count(self::$itemlist); + for ($key = count(self::$itemlist) - 1; $key >= 0; $key--) { + if (empty(self::$itemlist[$key]['contact-id'])) { + self::$itemlist[$key]['contact-id'] = $default_contact; + } else { + $default_contact = $item['contact-id']; + } + } + foreach (self::$itemlist AS $item) { + $found = dba::exists('item', array('uid' => $importer["uid"], 'uri' => $item["uri"])); + if ($found) { + logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already exists.", LOGGER_DEBUG); + } else { + $ret = item_store($item); + logger('Item was stored with return value '.$ret); + } + } + } + self::$itemlist = array(); + } + } + return true; + } + + private static function processPost($xpath, $entry, &$item, $importer) { + $item["uri"] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue; + $item["body"] = html2bbcode($xpath->query('atom:content/text()', $entry)->item(0)->nodeValue); + $item["object-type"] = $xpath->query('activity:object-type/text()', $entry)->item(0)->nodeValue; + if (($item["object-type"] == ACTIVITY_OBJ_BOOKMARK) || ($item["object-type"] == ACTIVITY_OBJ_EVENT)) { + $item["title"] = $xpath->query('atom:title/text()', $entry)->item(0)->nodeValue; + $item["body"] = $xpath->query('atom:summary/text()', $entry)->item(0)->nodeValue; + } elseif ($item["object-type"] == ACTIVITY_OBJ_QUESTION) { + $item["title"] = $xpath->query('atom:title/text()', $entry)->item(0)->nodeValue; + } + + $item["created"] = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue; + $item["edited"] = $xpath->query('atom:updated/text()', $entry)->item(0)->nodeValue; + $conversation = $xpath->query('ostatus:conversation/text()', $entry)->item(0)->nodeValue; + $item['conversation-uri'] = $conversation; + + $conv = $xpath->query('ostatus:conversation', $entry); + if (is_object($conv->item(0))) { + foreach ($conv->item(0)->attributes AS $attributes) { + if ($attributes->name == "ref") { + $item['conversation-uri'] = $attributes->textContent; + } + if ($attributes->name == "href") { + $item['conversation-href'] = $attributes->textContent; + } + } + } + + if (empty($item['conversation-href']) && !empty($item['conversation-uri'])) { + $item['conversation-href'] = $item['conversation-uri']; + } + + $related = ""; + + $inreplyto = $xpath->query('thr:in-reply-to', $entry); + if (is_object($inreplyto->item(0))) { + foreach ($inreplyto->item(0)->attributes AS $attributes) { + if ($attributes->name == "ref") { + $item["parent-uri"] = $attributes->textContent; + } + if ($attributes->name == "href") { + $related = $attributes->textContent; + } + } + } + + $georsspoint = $xpath->query('georss:point', $entry); + if (!empty($georsspoint) && ($georsspoint->length > 0)) { + $item["coord"] = $georsspoint->item(0)->nodeValue; + } + + $categories = $xpath->query('atom:category', $entry); + if ($categories) { + foreach ($categories AS $category) { + foreach ($category->attributes AS $attributes) { + if ($attributes->name == "term") { + $term = $attributes->textContent; + if (strlen($item["tag"])) { + $item["tag"] .= ','; + } + $item["tag"] .= "#[url=".System::baseUrl()."/search?tag=".$term."]".$term."[/url]"; + } + } + } + } + + $self = ''; + $add_body = ''; + + $links = $xpath->query('atom:link', $entry); + if ($links) { + $link_data = self::processLinks($links, $item); + $self = $link_data['self']; + $add_body = $link_data['add_body']; + } + + $repeat_of = ""; + + $notice_info = $xpath->query('statusnet:notice_info', $entry); + if ($notice_info && ($notice_info->length > 0)) { + foreach ($notice_info->item(0)->attributes AS $attributes) { + if ($attributes->name == "source") { + $item["app"] = strip_tags($attributes->textContent); + } + if ($attributes->name == "repeat_of") { + $repeat_of = $attributes->textContent; + } + } + } + // Is it a repeated post? + if (($repeat_of != "") || ($item["verb"] == ACTIVITY_SHARE)) { + $link_data = self::processRepeatedItem($xpath, $entry, $item, $importer); + if (!empty($link_data['add_body'])) { + $add_body .= $link_data['add_body']; + } + } + + $item["body"] .= $add_body; + + // Only add additional data when there is no picture in the post + if (!strstr($item["body"],'[/img]')) { + $item["body"] = add_page_info_to_body($item["body"]); + } + + // Mastodon Content Warning + if (($item["verb"] == ACTIVITY_POST) && $xpath->evaluate('boolean(atom:summary)', $entry)) { + $clear_text = $xpath->query('atom:summary/text()', $entry)->item(0)->nodeValue; + + $item["body"] = html2bbcode($clear_text) . '[spoiler]' . $item["body"] . '[/spoiler]'; + } + + if (isset($item["parent-uri"]) && ($related != '')) { + self::FetchRelated($related, $item["parent-uri"], $importer); + $item["type"] = 'remote-comment'; + $item["gravity"] = GRAVITY_COMMENT; + } else { + $item["parent-uri"] = $item["uri"]; + } + + if ($item['author-link'] != '') { + $item = store_conversation($item); + } + + self::$itemlist[] = $item; + } + + private static function fetchRelated($related, $related_uri, $importer) { + $condition = array('`item-uri` = ? AND `protocol` IN (?, ?)', $related_uri, PROTOCOL_DFRN, PROTOCOL_OSTATUS_SALMON); + $conversation = dba::select('conversation', array('source', 'protocol'), $condition, array('limit' => 1)); + if (dbm::is_result($conversation)) { + $stored = true; + $xml = $conversation['source']; + if (self::process($xml, $importer, $contact, $hub, $stored, false)) { + return; + } + if ($conversation['protocol'] == PROTOCOL_OSTATUS_SALMON) { + dba::delete('conversation', array('item-uri' => $related_uri)); + } + } + + $stored = false; + $related_data = z_fetch_url($related); + + if (!$related_data['success']) { + return; + } + + $xml = ''; + + if (stristr($related_data['header'], 'Content-Type: application/atom+xml')) { + $xml = $related_data['body']; + } + + if ($xml == '') { + $doc = new DOMDocument(); + if (!@$doc->loadHTML($related_data['body'])) { + return; + } + $xpath = new DomXPath($doc); + + $links = $xpath->query('//link'); + if ($links) { + foreach ($links AS $link) { + $attribute = self::read_attributes($link); + if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) { + $related_atom = z_fetch_url($attribute['href']); + + if ($related_atom['success']) { + $xml = $related_atom['body']; + } + } + } + } + } + + // Workaround for older GNU Social servers + if (($xml == '') && strstr($related, '/notice/')) { + $related_atom = z_fetch_url(str_replace('/notice/', '/api/statuses/show/', $related).',atom'); + + if ($related_atom['success']) { + $xml = $related_atom['body']; + } + } + + if ($xml != '') { + self::process($xml, $importer, $contact, $hub, $stored, false); + } + return; + } + + private static function processRepeatedItem($xpath, $entry, &$item, $importer) { + $activityobjects = $xpath->query('activity:object', $entry)->item(0); + + if (!is_object($activityobjects)) { + return array(); + } + + $link_data = array(); + + $orig_uri = $xpath->query('atom:id/text()', $activityobjects)->item(0)->nodeValue; + + $links = $xpath->query("atom:link", $activityobjects); + if ($links) { + $link_data = self::processLinks($links, $item); + } + + $orig_body = $xpath->query('atom:content/text()', $activityobjects)->item(0)->nodeValue; + $orig_created = $xpath->query('atom:published/text()', $activityobjects)->item(0)->nodeValue; + $orig_edited = $xpath->query('atom:updated/text()', $activityobjects)->item(0)->nodeValue; + + $orig_contact = $contact; + $orig_author = self::fetchauthor($xpath, $activityobjects, $importer, $orig_contact, false); + + $item["author-name"] = $orig_author["author-name"]; + $item["author-link"] = $orig_author["author-link"]; + $item["author-avatar"] = $orig_author["author-avatar"]; + + $item["body"] = html2bbcode($orig_body); + $item["created"] = $orig_created; + $item["edited"] = $orig_edited; + + $item["uri"] = $orig_uri; + + $item["verb"] = $xpath->query('activity:verb/text()', $activityobjects)->item(0)->nodeValue; + + $item["object-type"] = $xpath->query('activity:object-type/text()', $activityobjects)->item(0)->nodeValue; + + $inreplyto = $xpath->query('thr:in-reply-to', $activityobjects); + if (is_object($inreplyto->item(0))) { + foreach ($inreplyto->item(0)->attributes AS $attributes) { + if ($attributes->name == "ref") { + $item["parent-uri"] = $attributes->textContent; + } + } + } + + return $link_data; + } + + private static function processLinks($links, &$item) { + $link_data = array('add_body' => '', 'self' => ''); + + foreach ($links AS $link) { + $attribute = self::read_attributes($link); + + if (($attribute['rel'] != "") && ($attribute['href'] != "")) { + switch ($attribute['rel']) { + case "alternate": + $item["plink"] = $attribute['href']; + if (($item["object-type"] == ACTIVITY_OBJ_QUESTION) || + ($item["object-type"] == ACTIVITY_OBJ_EVENT)) { + $item["body"] .= add_page_info($attribute['href']); + } + break; + case "ostatus:conversation": + $link_data['conversation'] = $attribute['href']; + $item['conversation-href'] = $link_data['conversation']; + if (!isset($item['conversation-uri'])) { + $item['conversation-uri'] = $item['conversation-href']; + } + break; + case "enclosure": + $filetype = strtolower(substr($attribute['type'], 0, strpos($attribute['type'],'/'))); + if ($filetype == 'image') { + $link_data['add_body'] .= "\n[img]".$attribute['href'].'[/img]'; + } else { + if (strlen($item["attach"])) { + $item["attach"] .= ','; + } + if (!isset($attribute['length'])) { + $attribute['length'] = "0"; + } + $item["attach"] .= '[attach]href="'.$attribute['href'].'" length="'.$attribute['length'].'" type="'.$attribute['type'].'" title="'.$attribute['title'].'"[/attach]'; + } + break; + case "related": + if ($item["object-type"] != ACTIVITY_OBJ_BOOKMARK) { + if (!isset($item["parent-uri"])) { + $item["parent-uri"] = $attribute['href']; + } + $link_data['related'] = $attribute['href']; + } else { + $item["body"] .= add_page_info($attribute['href']); + } + break; + case "self": + if ($item["plink"] == '') { + $item["plink"] = $attribute['href']; + } + $link_data['self'] = $attribute['href']; + break; + } + } + } + return $link_data; + } + /** * @brief Fetches author data * @@ -252,1119 +762,6 @@ class ostatus { return $attribute; } - /** - * @brief Imports an XML string containing OStatus elements - * - * @param string $xml The XML - * @param array $importer user record of the importing user - * @param $contact - * @param array $hub Called by reference, returns the fetched hub data - */ - public static function import($xml,$importer,&$contact, &$hub) { - /// @todo this function is too long. It has to be split in many parts - - logger("Import OStatus message", LOGGER_DEBUG); - - if ($xml == "") { - return; - } - //$tempfile = tempnam(get_temppath(), "import"); - //file_put_contents($tempfile, $xml); - - $doc = new DOMDocument(); - @$doc->loadXML($xml); - - $xpath = new DomXPath($doc); - $xpath->registerNamespace('atom', NAMESPACE_ATOM1); - $xpath->registerNamespace('thr', NAMESPACE_THREAD); - $xpath->registerNamespace('georss', NAMESPACE_GEORSS); - $xpath->registerNamespace('activity', NAMESPACE_ACTIVITY); - $xpath->registerNamespace('media', NAMESPACE_MEDIA); - $xpath->registerNamespace('poco', NAMESPACE_POCO); - $xpath->registerNamespace('ostatus', NAMESPACE_OSTATUS); - $xpath->registerNamespace('statusnet', NAMESPACE_STATUSNET); - - $gub = ""; - $hub_attributes = $xpath->query("/atom:feed/atom:link[@rel='hub']")->item(0)->attributes; - if (is_object($hub_attributes)) { - foreach ($hub_attributes AS $hub_attribute) { - if ($hub_attribute->name == "href") { - $hub = $hub_attribute->textContent; - logger("Found hub ".$hub, LOGGER_DEBUG); - } - } - } - $header = array(); - $header["uid"] = $importer["uid"]; - $header["network"] = NETWORK_OSTATUS; - $header["type"] = "remote"; - $header["wall"] = 0; - $header["origin"] = 0; - $header["gravity"] = GRAVITY_PARENT; - - // it could either be a received post or a post we fetched by ourselves - // depending on that, the first node is different - $first_child = $doc->firstChild->tagName; - - if ($first_child == "feed") { - $entries = $xpath->query('/atom:feed/atom:entry'); - $header["protocol"] = PROTOCOL_OSTATUS_FEED; - } else { - $entries = $xpath->query('/atom:entry'); - $header["protocol"] = PROTOCOL_OSTATUS_SALMON; - } - $conversation = ""; - $conversationlist = array(); - $item_id = 0; - - // Reverse the order of the entries - $entrylist = array(); - - foreach ($entries AS $entry) { - $entrylist[] = $entry; - } - foreach (array_reverse($entrylist) AS $entry) { - - $mention = false; - - // fetch the author - if ($first_child == "feed") { - $author = self::fetchauthor($xpath, $doc->firstChild, $importer, $contact, false); - } else { - $author = self::fetchauthor($xpath, $entry, $importer, $contact, false); - } - $value = $xpath->evaluate('atom:author/poco:preferredUsername/text()', $context)->item(0)->nodeValue; - if ($value != "") { - $nickname = $value; - } else { - $nickname = $author["author-name"]; - } - $item = array_merge($header, $author); - - // Now get the item - $item["uri"] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue; - - $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s'", - intval($importer["uid"]), dbesc($item["uri"])); - if (dbm::is_result($r)) { - logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$r[0]["id"], LOGGER_DEBUG); - continue; - } - - $item["body"] = html2bbcode($xpath->query('atom:content/text()', $entry)->item(0)->nodeValue); - $item["object-type"] = $xpath->query('activity:object-type/text()', $entry)->item(0)->nodeValue; - $item["verb"] = $xpath->query('activity:verb/text()', $entry)->item(0)->nodeValue; - - if (($item["object-type"] == ACTIVITY_OBJ_BOOKMARK) || ($item["object-type"] == ACTIVITY_OBJ_EVENT)) { - $item["title"] = $xpath->query('atom:title/text()', $entry)->item(0)->nodeValue; - $item["body"] = $xpath->query('atom:summary/text()', $entry)->item(0)->nodeValue; - } elseif ($item["object-type"] == ACTIVITY_OBJ_QUESTION) { - $item["title"] = $xpath->query('atom:title/text()', $entry)->item(0)->nodeValue; - } - $item["source"] = $xml; - - /// @TODO - /// Delete a message - if ($item["verb"] == "qvitter-delete-notice" || $item["verb"] == ACTIVITY_DELETE) { - // ignore "Delete" messages (by now) - logger("Ignore delete message ".print_r($item, true)); - continue; - } - - if ($item["verb"] == ACTIVITY_JOIN) { - // ignore "Join" messages - logger("Ignore join message ".print_r($item, true)); - continue; - } - - if ($item["verb"] == ACTIVITY_FOLLOW) { - new_follower($importer, $contact, $item, $nickname); - continue; - } - - if ($item["verb"] == NAMESPACE_OSTATUS."/unfollow") { - lose_follower($importer, $contact, $item, $dummy); - continue; - } - - if ($item["verb"] == ACTIVITY_FAVORITE) { - $orig_uri = $xpath->query("activity:object/atom:id", $entry)->item(0)->nodeValue; - logger("Favorite ".$orig_uri." ".print_r($item, true)); - - $item["verb"] = ACTIVITY_LIKE; - $item["parent-uri"] = $orig_uri; - $item["gravity"] = GRAVITY_LIKE; - } - - if ($item["verb"] == NAMESPACE_OSTATUS."/unfavorite") { - // Ignore "Unfavorite" message - logger("Ignore unfavorite message ".print_r($item, true)); - continue; - } - - // http://activitystrea.ms/schema/1.0/rsvp-yes - if (!in_array($item["verb"], array(ACTIVITY_POST, ACTIVITY_LIKE, ACTIVITY_SHARE))) { - logger("Unhandled verb ".$item["verb"]." ".print_r($item, true)); - } - $item["created"] = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue; - $item["edited"] = $xpath->query('atom:updated/text()', $entry)->item(0)->nodeValue; - $conversation = $xpath->query('ostatus:conversation/text()', $entry)->item(0)->nodeValue; - $item['conversation-uri'] = $conversation; - - $conv = $xpath->query('ostatus:conversation', $entry); - if (is_object($conv->item(0))) { - foreach ($conv->item(0)->attributes AS $attributes) { - if ($attributes->name == "ref") { - $item['conversation-uri'] = $attributes->textContent; - } - if ($attributes->name == "href") { - $item['conversation-href'] = $attributes->textContent; - } - } - } - - $related = ""; - - $inreplyto = $xpath->query('thr:in-reply-to', $entry); - if (is_object($inreplyto->item(0))) { - foreach ($inreplyto->item(0)->attributes AS $attributes) { - if ($attributes->name == "ref") { - $item["parent-uri"] = $attributes->textContent; - } - if ($attributes->name == "href") { - $related = $attributes->textContent; - } - } - } - - $georsspoint = $xpath->query('georss:point', $entry); - if ($georsspoint) { - $item["coord"] = $georsspoint->item(0)->nodeValue; - } - $categories = $xpath->query('atom:category', $entry); - if ($categories) { - foreach ($categories AS $category) { - foreach ($category->attributes AS $attributes) { - if ($attributes->name == "term") { - $term = $attributes->textContent; - if (strlen($item["tag"])) { - $item["tag"] .= ','; - } - $item["tag"] .= "#[url=".System::baseUrl()."/search?tag=".$term."]".$term."[/url]"; - } - } - } - } - - $self = ""; - - $add_body = ""; - - $links = $xpath->query('atom:link', $entry); - if ($links) { - foreach ($links AS $link) { - $attribute = self::read_attributes($link); - - if (($attribute['rel'] != "") && ($attribute['href'] != "")) { - switch ($attribute['rel']) { - case "alternate": - $item["plink"] = $attribute['href']; - if (($item["object-type"] == ACTIVITY_OBJ_QUESTION) || - ($item["object-type"] == ACTIVITY_OBJ_EVENT)) { - $item["body"] .= add_page_info($attribute['href']); - } - break; - case "ostatus:conversation": - $conversation = $attribute['href']; - $item['conversation-href'] = $conversation; - if (!isset($item['conversation-uri'])) { - $item['conversation-uri'] = $item['conversation-href']; - } - break; - case "enclosure": - $filetype = strtolower(substr($attribute['type'], 0, strpos($attribute['type'],'/'))); - if ($filetype == 'image') { - $add_body .= "\n[img]".$attribute['href'].'[/img]'; - } else { - if (strlen($item["attach"])) { - $item["attach"] .= ','; - } - if (!isset($attribute['length'])) { - $attribute['length'] = "0"; - } - $item["attach"] .= '[attach]href="'.$attribute['href'].'" length="'.$attribute['length'].'" type="'.$attribute['type'].'" title="'.$attribute['title'].'"[/attach]'; - } - break; - case "related": - if ($item["object-type"] != ACTIVITY_OBJ_BOOKMARK) { - if (!isset($item["parent-uri"])) { - $item["parent-uri"] = $attribute['href']; - } - if ($related == "") { - $related = $attribute['href']; - } - } else { - $item["body"] .= add_page_info($attribute['href']); - } - break; - case "self": - $self = $attribute['href']; - break; - case "mentioned": - // Notification check - if ($importer["nurl"] == normalise_link($attribute['href'])) { - $mention = true; - } - break; - } - } - } - } - - $local_id = ""; - $repeat_of = ""; - - $notice_info = $xpath->query('statusnet:notice_info', $entry); - if ($notice_info && ($notice_info->length > 0)) { - foreach ($notice_info->item(0)->attributes AS $attributes) { - if ($attributes->name == "source") { - $item["app"] = strip_tags($attributes->textContent); - } - if ($attributes->name == "local_id") { - $local_id = $attributes->textContent; - } - if ($attributes->name == "repeat_of") { - $repeat_of = $attributes->textContent; - } - } - } - - // Is it a repeated post? - if (($repeat_of != "") || ($item["verb"] == ACTIVITY_SHARE)) { - $activityobjects = $xpath->query('activity:object', $entry)->item(0); - - if (is_object($activityobjects)) { - - $orig_uri = $xpath->query("activity:object/atom:id", $activityobjects)->item(0)->nodeValue; - if (!isset($orig_uri)) { - $orig_uri = $xpath->query('atom:id/text()', $activityobjects)->item(0)->nodeValue; - } - $orig_links = $xpath->query("activity:object/atom:link[@rel='alternate']", $activityobjects); - if ($orig_links && ($orig_links->length > 0)) { - foreach ($orig_links->item(0)->attributes AS $attributes) { - if ($attributes->name == "href") { - $orig_link = $attributes->textContent; - } - } - } - if (!isset($orig_link)) { - $orig_link = $xpath->query("atom:link[@rel='alternate']", $activityobjects)->item(0)->nodeValue; - } - if (!isset($orig_link)) { - $orig_link = self::convert_href($orig_uri); - } - $orig_body = $xpath->query('activity:object/atom:content/text()', $activityobjects)->item(0)->nodeValue; - if (!isset($orig_body)) { - $orig_body = $xpath->query('atom:content/text()', $activityobjects)->item(0)->nodeValue; - } - $orig_created = $xpath->query('atom:published/text()', $activityobjects)->item(0)->nodeValue; - $orig_edited = $xpath->query('atom:updated/text()', $activityobjects)->item(0)->nodeValue; - - $orig_contact = $contact; - $orig_author = self::fetchauthor($xpath, $activityobjects, $importer, $orig_contact, false); - - $item["author-name"] = $orig_author["author-name"]; - $item["author-link"] = $orig_author["author-link"]; - $item["author-avatar"] = $orig_author["author-avatar"]; - - $item["body"] = html2bbcode($orig_body); - $item["created"] = $orig_created; - $item["edited"] = $orig_edited; - - $item["uri"] = $orig_uri; - - if (!isset($item["plink"])) { - $item["plink"] = $orig_link; - } - - $item["verb"] = $xpath->query('activity:verb/text()', $activityobjects)->item(0)->nodeValue; - - $item["object-type"] = $xpath->query('activity:object/activity:object-type/text()', $activityobjects)->item(0)->nodeValue; - if (!isset($item["object-type"])) { - $item["object-type"] = $xpath->query('activity:object-type/text()', $activityobjects)->item(0)->nodeValue; - } - - $enclosures = $xpath->query("atom:link[@rel='alternate']", $activityobjects); - if ($enclosures) { - foreach ($enclosures AS $link) { - $attribute = self::read_attributes($link); - if ($href != "") { - if (strlen($item["attach"])) { - $item["attach"] .= ','; - } - if (!isset($attribute['length'])) { - $attribute['length'] = "0"; - } - $item["attach"] .= '[attach]href="'.$attribute['href'].'" length="'.$attribute['length'].'" type="'.$attribute['type'].'" title="'.$attribute['title'].'"[/attach]'; - } - } - } - } - } - - $item["body"] .= $add_body; - - // Only add additional data when there is no picture in the post - if (!strstr($item["body"],'[/img]')) { - $item["body"] = add_page_info_to_body($item["body"]); - } - - // Mastodon Content Warning - if (($item["verb"] == ACTIVITY_POST) && $xpath->evaluate('boolean(atom:summary)', $entry)) { - $clear_text = $xpath->query('atom:summary/text()', $entry)->item(0)->nodeValue; - - $item["body"] = html2bbcode($clear_text) . '[spoiler]' . $item["body"] . '[/spoiler]'; - } - - if (isset($item["parent-uri"])) { - $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s'", - intval($importer["uid"]), dbesc($item["parent-uri"])); - - // Only fetch missing stuff if it is a comment or reshare. - if (in_array($item["verb"], array(ACTIVITY_POST, ACTIVITY_SHARE)) && - !dbm::is_result($r) && ($related != "")) { - $reply_path = str_replace("/notice/", "/api/statuses/show/", $related).".atom"; - - if ($reply_path != $related) { - logger("Fetching related items for user ".$importer["uid"]." from ".$reply_path, LOGGER_DEBUG); - $reply_xml = fetch_url($reply_path); - - $reply_contact = $contact; - self::import($reply_xml,$importer,$reply_contact, $reply_hub); - - // After the import try to fetch the parent item again - $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s'", - intval($importer["uid"]), dbesc($item["parent-uri"])); - } - } - if (dbm::is_result($r)) { - $item["type"] = 'remote-comment'; - $item["gravity"] = GRAVITY_COMMENT; - } - } else { - $item["parent-uri"] = $item["uri"]; - } - $item_id = self::completion($conversation, $importer["uid"], $item, $self); - - if (!$item_id) { - // Store the conversation data. This is normally done in "item_store" - // but since something went wrong, we want to be sure to save the data. - store_conversation($item); - logger("Error storing item", LOGGER_DEBUG); - continue; - } - - logger("Item was stored with id ".$item_id, LOGGER_DEBUG); - } - } - - /** - * @brief Create an url out of an uri - * - * @param string $href URI in the format "parameter1:parameter1:..." - * - * @return string URL in the format http(s)://.... - */ - public static function convert_href($href) { - $elements = explode(":",$href); - - if ((count($elements) <= 2) || ($elements[0] != "tag")) - return $href; - - $server = explode(",", $elements[1]); - $conversation = explode("=", $elements[2]); - - if ((count($elements) == 4) && ($elements[2] == "post")) - return "http://".$server[0]."/notice/".$elements[3]; - - if ((count($conversation) != 2) || ($conversation[1] =="")) { - return $href; - } - if ($elements[3] == "objectType=thread") { - return "http://".$server[0]."/conversation/".$conversation[1]; - } else { - return "http://".$server[0]."/notice/".$conversation[1]; - } - return $href; - } - - /** - * @brief Checks if there are entries in conversations that aren't present on our side - * - * @param bool $mentions Fetch conversations where we are mentioned - * @param bool $override Override the interval setting - */ - public static function check_conversations($mentions = false, $override = false) { - $last = get_config('system','ostatus_last_poll'); - - $poll_interval = intval(get_config('system','ostatus_poll_interval')); - if (!$poll_interval) { - $poll_interval = self::OSTATUS_DEFAULT_POLL_INTERVAL; - } - - // Don't poll if the interval is set negative - if (($poll_interval < 0) && !$override) { - return; - } - - if (!$mentions) { - $poll_timeframe = intval(get_config('system','ostatus_poll_timeframe')); - if (!$poll_timeframe) { - $poll_timeframe = self::OSTATUS_DEFAULT_POLL_TIMEFRAME; - } - } else { - $poll_timeframe = intval(get_config('system','ostatus_poll_timeframe')); - if (!$poll_timeframe) { - $poll_timeframe = self::OSTATUS_DEFAULT_POLL_TIMEFRAME_MENTIONS; - } - } - - - if ($last && !$override) { - $next = $last + ($poll_interval * 60); - if ($next > time()) { - logger('poll interval not reached'); - return; - } - } - - logger('cron_start'); - - $start = date("Y-m-d H:i:s", time() - ($poll_timeframe * 60)); - - if ($mentions) { - $conversations = q("SELECT `term`.`oid`, `term`.`url`, `term`.`uid` FROM `term` - STRAIGHT_JOIN `thread` ON `thread`.`iid` = `term`.`oid` AND `thread`.`uid` = `term`.`uid` - WHERE `term`.`type` = 7 AND `term`.`term` > '%s' AND `thread`.`mention` - GROUP BY `term`.`url`, `term`.`uid`, `term`.`oid`, `term`.`term` ORDER BY `term`.`term` DESC", dbesc($start)); - } else { - $conversations = q("SELECT `oid`, `url`, `uid` FROM `term` - WHERE `type` = 7 AND `term` > '%s' - GROUP BY `url`, `uid`, `oid`, `term` ORDER BY `term` DESC", dbesc($start)); - } - - foreach ($conversations AS $conversation) { - self::completion($conversation['url'], $conversation['uid']); - } - - logger('cron_end'); - - set_config('system','ostatus_last_poll', time()); - } - - /** - * @brief Updates the gcontact table with actor data from the conversation - * - * @param object $actor The actor object that contains the contact data - */ - private static function conv_fetch_actor($actor) { - - // We set the generation to "3" since the data here is not as reliable as the data we get on other occasions - $contact = array("network" => NETWORK_OSTATUS, "generation" => 3); - - if (isset($actor->url)) - $contact["url"] = $actor->url; - - if (isset($actor->displayName)) - $contact["name"] = $actor->displayName; - - if (isset($actor->portablecontacts_net->displayName)) - $contact["name"] = $actor->portablecontacts_net->displayName; - - if (isset($actor->portablecontacts_net->preferredUsername)) - $contact["nick"] = $actor->portablecontacts_net->preferredUsername; - - if (isset($actor->id)) - $contact["alias"] = $actor->id; - - if (isset($actor->summary)) - $contact["about"] = $actor->summary; - - if (isset($actor->portablecontacts_net->note)) - $contact["about"] = $actor->portablecontacts_net->note; - - if (isset($actor->portablecontacts_net->addresses->formatted)) - $contact["location"] = $actor->portablecontacts_net->addresses->formatted; - - - if (isset($actor->image->url)) - $contact["photo"] = $actor->image->url; - - if (isset($actor->image->width)) - $avatarwidth = $actor->image->width; - - if (is_array($actor->status_net->avatarLinks)) - foreach ($actor->status_net->avatarLinks AS $avatar) { - if ($avatarsize < $avatar->width) { - $contact["photo"] = $avatar->url; - $avatarsize = $avatar->width; - } - } - - $contact["hide"] = false; // OStatus contacts are never hidden - update_gcontact($contact); - } - - /** - * @brief Fetches the conversation url for a given item link or conversation id - * - * @param string $self The link to the posting - * @param string $conversation_id The conversation id - * - * @return string The conversation url - */ - private static function fetch_conversation($self, $conversation_id = "") { - - if ($conversation_id != "") { - $elements = explode(":", $conversation_id); - - if ((count($elements) <= 2) || ($elements[0] != "tag")) - return $conversation_id; - } - - if ($self == "") - return ""; - - $json = str_replace(".atom", ".json", $self); - - $raw = fetch_url($json); - if ($raw == "") - return ""; - - $data = json_decode($raw); - if (!is_object($data)) - return ""; - - $conversation_id = $data->statusnet_conversation_id; - - $pos = strpos($self, "/api/statuses/show/"); - $base_url = substr($self, 0, $pos); - - return $base_url."/conversation/".$conversation_id; - } - - /** - * @brief Fetches a shared object from a given conversation object - * - * Sometimes GNU Social seems to fail when returning shared objects. - * Then they don't contains all needed data. - * We then try to find this object in the conversation - * - * @param string $id Message id - * @param object $conversation Conversation object - * - * @return object The shared object - */ - private static function shared_object($id, $conversation) { - if (!is_array($conversation->items)) { - return false; - } - foreach ($conversation->items AS $single_conv) { - if ($single_conv->id == $id) { - return $single_conv; - } - } - return false; - } - - /** - * @brief Fetches actor details of a given actor and user id - * - * @param string $actor The actor url - * @param int $uid The user id - * @param int $contact_id The default contact-id - * - * @return array Array with actor details - */ - private static function get_actor_details($actor, $uid, $contact_id) { - - $details = array(); - - $contact = q("SELECT `id`, `rel`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'", - $uid, normalise_link($actor), NETWORK_STATUSNET); - - if (!$contact) - $contact = q("SELECT `id`, `rel`, `network` FROM `contact` WHERE `uid` = %d AND `alias` IN ('%s', '%s') AND `network` != '%s'", - $uid, $actor, normalise_link($actor), NETWORK_STATUSNET); - - if ($contact) { - logger("Found contact for url ".$actor, LOGGER_DEBUG); - $details["contact_id"] = $contact[0]["id"]; - $details["network"] = $contact[0]["network"]; - - $details["not_following"] = !in_array($contact[0]["rel"], array(CONTACT_IS_SHARING, CONTACT_IS_FRIEND)); - } else { - logger("No contact found for user ".$uid." and url ".$actor, LOGGER_DEBUG); - - // Adding a global contact - /// @TODO Use this data for the post - $details["global_contact_id"] = get_contact($actor, 0); - - logger("Global contact ".$global_contact_id." found for url ".$actor, LOGGER_DEBUG); - - $details["contact_id"] = $contact_id; - $details["network"] = NETWORK_OSTATUS; - - $details["not_following"] = true; - } - - return $details; - } - - /** - * @brief Stores an item and completes the thread - * - * @param string $conversation_url The URI of the conversation - * @param integer $uid The user id - * @param array $item Data of the item that is to be posted - * - * @return integer The item id of the posted item array - */ - private static function completion($conversation_url, $uid, $item = array(), $self = "") { - - /// @todo This function is totally ugly and has to be rewritten totally - - // Import all threads or only threads that were started by our followers? - $all_threads = !get_config('system','ostatus_full_threads'); - - $item_stored = -1; - - $conversation_url = self::fetch_conversation($self, $conversation_url); - - // If the thread shouldn't be completed then store the item and go away - // Don't do a completion on liked content - if (((intval(get_config('system','ostatus_poll_interval')) == -2) && (count($item) > 0)) || - ($item["verb"] == ACTIVITY_LIKE) || ($conversation_url == "")) { - $item_stored = item_store($item, $all_threads); - return $item_stored; - } - - // Get the parent - $parents = q("SELECT `item`.`id`, `item`.`parent`, `item`.`uri`, `item`.`contact-id`, `item`.`type`, - `item`.`verb`, `item`.`visible` FROM `term` - STRAIGHT_JOIN `item` AS `thritem` ON `thritem`.`parent` = `term`.`oid` - STRAIGHT_JOIN `item` ON `item`.`parent` = `thritem`.`parent` - WHERE `term`.`uid` = %d AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`url` = '%s'", - intval($uid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($conversation_url)); - if ($parents) { - $parent = $parents[0]; - } elseif (count($item) > 0) { - $parent = $item; - $parent["type"] = "remote"; - $parent["verb"] = ACTIVITY_POST; - $parent["visible"] = 1; - } else { - // Preset the parent - $r = q("SELECT `id` FROM `contact` WHERE `self` AND `uid`=%d", $uid); - if (!dbm::is_result($r)) { - return -2; - } - $parent = array(); - $parent["id"] = 0; - $parent["parent"] = 0; - $parent["uri"] = ""; - $parent["contact-id"] = $r[0]["id"]; - $parent["type"] = "remote"; - $parent["verb"] = ACTIVITY_POST; - $parent["visible"] = 1; - } - - $conv = str_replace("/conversation/", "/api/statusnet/conversation/", $conversation_url).".as"; - $pageno = 1; - $items = array(); - - logger('fetching conversation url '.$conv.' (Self: '.$self.') for user '.$uid); - - do { - $conv_arr = z_fetch_url($conv."?page=".$pageno); - - // If it is a non-ssl site and there is an error, then try ssl or vice versa - if (!$conv_arr["success"] && (substr($conv, 0, 7) == "http://")) { - $conv = str_replace("http://", "https://", $conv); - $conv_as = fetch_url($conv."?page=".$pageno); - } elseif (!$conv_arr["success"] && (substr($conv, 0, 8) == "https://")) { - $conv = str_replace("https://", "http://", $conv); - $conv_as = fetch_url($conv."?page=".$pageno); - } else { - $conv_as = $conv_arr["body"]; - } - $conv_as = str_replace(',"statusnet:notice_info":', ',"statusnet_notice_info":', $conv_as); - $conv_as = json_decode($conv_as); - - $no_of_items = sizeof($items); - - if (@is_array($conv_as->items)) - foreach ($conv_as->items AS $single_item) - $items[$single_item->id] = $single_item; - - if ($no_of_items == sizeof($items)) - break; - - $pageno++; - - } while (true); - - logger('fetching conversation done. Found '.count($items).' items'); - - if (!sizeof($items)) { - if (count($item) > 0) { - $item_stored = item_store($item, $all_threads); - - if ($item_stored) { - logger("Conversation ".$conversation_url." couldn't be fetched. Item uri ".$item["uri"]." stored: ".$item_stored, LOGGER_DEBUG); - self::store_conversation($item_id, $conversation_url); - } - - return $item_stored; - } else { - return -3; - } - } - - $items = array_reverse($items); - - $r = q("SELECT `nurl` FROM `contact` WHERE `uid` = %d AND `self`", intval($uid)); - $importer = $r[0]; - - $new_parent = true; - - foreach ($items as $single_conv) { - - // Update the gcontact table - self::conv_fetch_actor($single_conv->actor); - - // Test - remove before flight - //$tempfile = tempnam(get_temppath(), "conversation"); - //file_put_contents($tempfile, json_encode($single_conv)); - - $mention = false; - - if (isset($single_conv->object->id)) - $single_conv->id = $single_conv->object->id; - - $plink = self::convert_href($single_conv->id); - if (isset($single_conv->object->url)) - $plink = self::convert_href($single_conv->object->url); - - if (@!$single_conv->id) - continue; - - logger("Got id ".$single_conv->id, LOGGER_DEBUG); - - if ($first_id == "") { - $first_id = $single_conv->id; - - // The first post of the conversation isn't our first post. There are three options: - // 1. Our conversation hasn't the "real" thread starter - // 2. This first post is a post inside our thread - // 3. This first post is a post inside another thread - if (($first_id != $parent["uri"]) && ($parent["uri"] != "")) { - - $new_parent = true; - - $new_parents = q("SELECT `id`, `parent`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `id` IN - (SELECT `parent` FROM `item` - WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s')) LIMIT 1", - intval($uid), dbesc($first_id), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); - if ($new_parents) { - if ($new_parents[0]["parent"] == $parent["parent"]) { - // Option 2: This post is already present inside our thread - but not as thread starter - logger("Option 2: uri present in our thread: ".$first_id, LOGGER_DEBUG); - $first_id = $parent["uri"]; - } else { - // Option 3: Not so good. We have mixed parents. We have to see how to clean this up. - // For now just take the new parent. - $parent = $new_parents[0]; - $first_id = $parent["uri"]; - logger("Option 3: mixed parents for uri ".$first_id, LOGGER_DEBUG); - } - } else { - // Option 1: We hadn't got the real thread starter - // We have to clean up our existing messages. - $parent["id"] = 0; - $parent["uri"] = $first_id; - logger("Option 1: we have a new parent: ".$first_id, LOGGER_DEBUG); - } - } elseif ($parent["uri"] == "") { - $parent["id"] = 0; - $parent["uri"] = $first_id; - } - } - - $parent_uri = $parent["uri"]; - - // "context" only seems to exist on older servers - if (isset($single_conv->context->inReplyTo->id)) { - $parent_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", - intval($uid), dbesc($single_conv->context->inReplyTo->id), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); - if ($parent_exists) - $parent_uri = $single_conv->context->inReplyTo->id; - } - - // This is the current way - if (isset($single_conv->object->inReplyTo->id)) { - $parent_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", - intval($uid), dbesc($single_conv->object->inReplyTo->id), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); - if ($parent_exists) - $parent_uri = $single_conv->object->inReplyTo->id; - } - - $message_exists = q("SELECT `id`, `parent`, `uri` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", - intval($uid), dbesc($single_conv->id), - dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); - if ($message_exists) { - logger("Message ".$single_conv->id." already existed on the system", LOGGER_DEBUG); - - if ($parent["id"] != 0) { - $existing_message = $message_exists[0]; - - // We improved the way we fetch OStatus messages, this shouldn't happen very often now - /// @TODO We have to change the shadow copies as well. This way here is really ugly. - if ($existing_message["parent"] != $parent["id"]) { - logger('updating id '.$existing_message["id"].' with parent '.$existing_message["parent"].' to parent '.$parent["id"].' uri '.$parent["uri"].' thread '.$parent_uri, LOGGER_DEBUG); - - // Update the parent id of the selected item - $r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s' WHERE `id` = %d", - intval($parent["id"]), dbesc($parent["uri"]), intval($existing_message["id"])); - - // Update the parent uri in the thread - but only if it points to itself - $r = q("UPDATE `item` SET `thr-parent` = '%s' WHERE `id` = %d AND `uri` = `thr-parent`", - dbesc($parent_uri), intval($existing_message["id"])); - - // try to change all items of the same parent - $r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s' WHERE `parent` = %d", - intval($parent["id"]), dbesc($parent["uri"]), intval($existing_message["parent"])); - - // Update the parent uri in the thread - but only if it points to itself - $r = q("UPDATE `item` SET `thr-parent` = '%s' WHERE (`parent` = %d) AND (`uri` = `thr-parent`)", - dbesc($parent["uri"]), intval($existing_message["parent"])); - - // Now delete the thread - delete_thread($existing_message["parent"]); - } - } - - // The item we are having on the system is the one that we wanted to store via the item array - if (isset($item["uri"]) && ($item["uri"] == $existing_message["uri"])) { - $item = array(); - $item_stored = 0; - } - - continue; - } - - if (is_array($single_conv->to)) - foreach ($single_conv->to AS $to) - if ($importer["nurl"] == normalise_link($to->id)) - $mention = true; - - $actor = $single_conv->actor->id; - if (isset($single_conv->actor->url)) - $actor = $single_conv->actor->url; - - $details = self::get_actor_details($actor, $uid, $parent["contact-id"]); - - // Do we only want to import threads that were started by our contacts? - if ($details["not_following"] && $new_parent && get_config('system','ostatus_full_threads')) { - logger("Don't import uri ".$first_id." because user ".$uid." doesn't follow the person ".$actor, LOGGER_DEBUG); - continue; - } - - /// @TODO One statment is okay (until if () ) - $arr = array(); - $arr["network"] = $details["network"]; - $arr["uri"] = $single_conv->id; - $arr["plink"] = $plink; - $arr["uid"] = $uid; - $arr["contact-id"] = $details["contact_id"]; - $arr["parent-uri"] = $parent_uri; - $arr["created"] = $single_conv->published; - $arr["edited"] = $single_conv->published; - $arr["owner-name"] = $single_conv->actor->displayName; - if ($arr["owner-name"] == '') - $arr["owner-name"] = $single_conv->actor->contact->displayName; - if ($arr["owner-name"] == '') - $arr["owner-name"] = $single_conv->actor->portablecontacts_net->displayName; - - $arr["owner-link"] = $actor; - $arr["owner-avatar"] = Probe::fixAvatar($single_conv->actor->image->url, $arr["owner-link"]); - - $arr["author-name"] = $arr["owner-name"]; - $arr["author-link"] = $arr["owner-link"]; - $arr["author-avatar"] = $arr["owner-avatar"]; - $arr["body"] = add_page_info_to_body(html2bbcode($single_conv->content)); - - if (isset($single_conv->status_net->conversation)) { - $arr['conversation-uri'] = $single_conv->status_net->conversation; - } - - if (isset($single_conv->status_net->notice_info->source)) { - $arr["app"] = strip_tags($single_conv->status_net->notice_info->source); - } elseif (isset($single_conv->statusnet->notice_info->source)) { - $arr["app"] = strip_tags($single_conv->statusnet->notice_info->source); - } elseif (isset($single_conv->statusnet_notice_info->source)) { - $arr["app"] = strip_tags($single_conv->statusnet_notice_info->source); - } elseif (isset($single_conv->provider->displayName)) { - $arr["app"] = $single_conv->provider->displayName; - } else { - $arr["app"] = "OStatus"; - } - - $arr["source"] = json_encode($single_conv); - $arr["protocol"] = PROTOCOL_GS_CONVERSATION; - - $arr["verb"] = $parent["verb"]; - $arr["visible"] = $parent["visible"]; - $arr["location"] = $single_conv->location->displayName; - $arr["coord"] = trim($single_conv->location->lat." ".$single_conv->location->lon); - - // Is it a reshared item? - if (isset($single_conv->verb) && ($single_conv->verb == "share") && isset($single_conv->object)) { - if (is_array($single_conv->object)) - $single_conv->object = $single_conv->object[0]; - - // Sometimes GNU Social doesn't returns a complete object - if (!isset($single_conv->object->actor->url)) { - $object = self::shared_object($single_conv->object->id, $conversation); - if (is_object($object)) { - $single_conv->object = $object; - } - } - - if (isset($single_conv->object->actor->url)) { - logger("Found reshared item ".$single_conv->object->id); - - // $single_conv->object->context->conversation; - - if (isset($single_conv->object->object->id)) { - $arr["uri"] = $single_conv->object->object->id; - } else { - $arr["uri"] = $single_conv->object->id; - } - if (isset($single_conv->object->object->url)) { - $plink = self::convert_href($single_conv->object->object->url); - } else { - $plink = self::convert_href($single_conv->object->url); - } - if (isset($single_conv->object->object->content)) { - $arr["body"] = add_page_info_to_body(html2bbcode($single_conv->object->object->content)); - } else { - $arr["body"] = add_page_info_to_body(html2bbcode($single_conv->object->content)); - } - $arr["plink"] = $plink; - - $arr["created"] = $single_conv->object->published; - $arr["edited"] = $single_conv->object->published; - - $arr["author-name"] = $single_conv->object->actor->displayName; - if ($arr["owner-name"] == '') { - $arr["author-name"] = $single_conv->object->actor->contact->displayName; - } - $arr["author-link"] = $single_conv->object->actor->url; - $arr["author-avatar"] = Probe::fixAvatar($single_conv->object->actor->image->url, $arr["author-link"]); - - $arr["app"] = $single_conv->object->provider->displayName."#"; - //$arr["verb"] = $single_conv->object->verb; - - $arr["location"] = $single_conv->object->location->displayName; - $arr["coord"] = trim($single_conv->object->location->lat." ".$single_conv->object->location->lon); - } - } - - if ($arr["location"] == "") - unset($arr["location"]); - - if ($arr["coord"] == "") - unset($arr["coord"]); - - // Copy fields from given item array - if (isset($item["uri"]) && (($item["uri"] == $arr["uri"]) || ($item["uri"] == $single_conv->id))) { - logger('Use stored item array for item with URI '.$item["uri"], LOGGER_DEBUG); - $newitem = item_store($item); - $item = array(); - $item_stored = $newitem; - } else { - $newitem = item_store($arr); - } - - if (!$newitem) { - logger("Item wasn't stored ".print_r($arr, true), LOGGER_DEBUG); - continue; - } - - logger('Stored new item '.$plink.' for parent '.$arr["parent-uri"].' under id '.$newitem, LOGGER_DEBUG); - - // Add the conversation entry (but don't fetch the whole conversation) - self::store_conversation($newitem, $conversation_url); - - // If the newly created item is the top item then change the parent settings of the thread - // This shouldn't happen anymore. This is supposed to be absolote. - if ($arr["uri"] == $first_id) { - logger('setting new parent to id '.$newitem); - $new_parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `id` = %d LIMIT 1", - intval($uid), intval($newitem)); - if ($new_parents) - $parent = $new_parents[0]; - } - } - - if (($item_stored < 0) && (count($item) > 0)) { - - if (get_config('system','ostatus_full_threads')) { - $details = self::get_actor_details($item["owner-link"], $uid, $item["contact-id"]); - if ($details["not_following"]) { - logger("Don't import uri ".$item["uri"]." because user ".$uid." doesn't follow the person ".$item["owner-link"], LOGGER_DEBUG); - return false; - } - } - - $item_stored = item_store($item, $all_threads); - if ($item_stored) { - logger("Uri ".$item["uri"]." wasn't found in conversation ".$conversation_url, LOGGER_DEBUG); - self::store_conversation($item_stored, $conversation_url); - } - } - - return $item_stored; - } - - /** - * @brief Stores conversation data into the database - * - * @param integer $itemid The id of the item - * @param string $conversation_url The uri of the conversation - */ - private static function store_conversation($itemid, $conversation_url) { - - $conversation_url = self::convert_href($conversation_url); - - $messages = q("SELECT `uid`, `parent`, `created`, `received`, `guid` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid)); - if (!$messages) - return; - $message = $messages[0]; - - // Store conversation url if not done before - $conversation = q("SELECT `url` FROM `term` WHERE `uid` = %d AND `oid` = %d AND `otype` = %d AND `type` = %d", - intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION)); - - if (!$conversation) { - $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `created`, `received`, `guid`) VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')", - intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), - dbesc($message["created"]), dbesc($conversation_url), dbesc($message["created"]), dbesc($message["received"]), dbesc($message["guid"])); - logger('Storing conversation url '.$conversation_url.' for id '.$itemid); - } - } - /** * @brief Checks if the current post is a reshare * diff --git a/mod/display.php b/mod/display.php index 56f7aa644..d8cdf658a 100644 --- a/mod/display.php +++ b/mod/display.php @@ -293,7 +293,7 @@ function display_content(App $a, $update = 0) { } // We are displaying an "alternate" link if that post was public. See issue 2864 - $is_public = dba::exists('item', array('id' => $item_id, 'private' => false, 'wall' => true)); + $is_public = dba::exists('item', array('id' => $item_id, 'private' => false)); if ($is_public) { $alternate = System::baseUrl().'/display/'.$nick.'/'.$item_id.'.atom'; } else { From c98c157efe3b281b308e5076e49515b1ea2a90b5 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 10 Sep 2017 07:23:14 +0000 Subject: [PATCH 2/9] Only some tabs --- include/ostatus.php | 56 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/include/ostatus.php b/include/ostatus.php index 124f421bf..78175562e 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -33,28 +33,28 @@ class ostatus { const OSTATUS_DEFAULT_POLL_TIMEFRAME = 1440; // given in minutes const OSTATUS_DEFAULT_POLL_TIMEFRAME_MENTIONS = 14400; // given in minutes - private static $itemlist; + private static $itemlist; - /** - * @brief Imports an XML string containing OStatus elements - * - * @param string $xml The XML - * @param array $importer user record of the importing user - * @param $contact - * @param array $hub Called by reference, returns the fetched hub data - */ + /** + * @brief Imports an XML string containing OStatus elements + * + * @param string $xml The XML + * @param array $importer user record of the importing user + * @param $contact + * @param array $hub Called by reference, returns the fetched hub data + */ public static function import($xml, $importer, &$contact, &$hub) { self::process($xml, $importer, $contact, $hub); } - /** - * @brief Imports an XML string containing OStatus elements - * - * @param string $xml The XML - * @param array $importer user record of the importing user - * @param $contact - * @param array $hub Called by reference, returns the fetched hub data - */ + /** + * @brief Imports an XML string containing OStatus elements + * + * @param string $xml The XML + * @param array $importer user record of the importing user + * @param $contact + * @param array $hub Called by reference, returns the fetched hub data + */ private static function process($xml, $importer, &$contact, &$hub, $stored = false, $initialize = true) { if ($initialize) { self::$itemlist = array(); @@ -78,16 +78,16 @@ class ostatus { $xpath->registerNamespace('ostatus', NAMESPACE_OSTATUS); $xpath->registerNamespace('statusnet', NAMESPACE_STATUSNET); - $hub = ""; - $hub_attributes = $xpath->query("/atom:feed/atom:link[@rel='hub']")->item(0)->attributes; - if (is_object($hub_attributes)) { - foreach ($hub_attributes AS $hub_attribute) { - if ($hub_attribute->name == "href") { - $hub = $hub_attribute->textContent; - logger("Found hub ".$hub, LOGGER_DEBUG); - } - } - } + $hub = ""; + $hub_attributes = $xpath->query("/atom:feed/atom:link[@rel='hub']")->item(0)->attributes; + if (is_object($hub_attributes)) { + foreach ($hub_attributes AS $hub_attribute) { + if ($hub_attribute->name == "href") { + $hub = $hub_attribute->textContent; + logger("Found hub ".$hub, LOGGER_DEBUG); + } + } + } $header = array(); $header["uid"] = $importer["uid"]; @@ -233,7 +233,7 @@ class ostatus { foreach (self::$itemlist AS $item) { $found = dba::exists('item', array('uid' => $importer["uid"], 'uri' => $item["uri"])); if ($found) { - logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already exists.", LOGGER_DEBUG); + logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already exists.", LOGGER_DEBUG); } else { $ret = item_store($item); logger('Item was stored with return value '.$ret); From 746169eabfa119c2db7e715c90f59df7ef10ca8a Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 10 Sep 2017 07:29:24 +0000 Subject: [PATCH 3/9] Improve comparabilty --- include/ostatus.php | 438 ++++++++++++++++++++++---------------------- 1 file changed, 219 insertions(+), 219 deletions(-) diff --git a/include/ostatus.php b/include/ostatus.php index 78175562e..d05e575f0 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -35,6 +35,225 @@ class ostatus { private static $itemlist; + /** + * @brief Fetches author data + * + * @param object $xpath The xpath object + * @param object $context The xml context of the author detals + * @param array $importer user record of the importing user + * @param array $contact Called by reference, will contain the fetched contact + * @param bool $onlyfetch Only fetch the header without updating the contact entries + * + * @return array Array of author related entries for the item + */ + private static function fetchauthor($xpath, $context, $importer, &$contact, $onlyfetch) { + + $author = array(); + $author["author-link"] = $xpath->evaluate('atom:author/atom:uri/text()', $context)->item(0)->nodeValue; + $author["author-name"] = $xpath->evaluate('atom:author/atom:name/text()', $context)->item(0)->nodeValue; + $addr = $xpath->evaluate('atom:author/atom:email/text()', $context)->item(0)->nodeValue; + + $aliaslink = $author["author-link"]; + + $alternate = $xpath->query("atom:author/atom:link[@rel='alternate']", $context)->item(0)->attributes; + if (is_object($alternate)) { + foreach ($alternate AS $attributes) { + if (($attributes->name == "href") && ($attributes->textContent != "")) { + $author["author-link"] = $attributes->textContent; + } + } + } + + $author["contact-id"] = $contact["id"]; + + if ($author["author-link"] != "") { + if ($aliaslink == "") { + $aliaslink = $author["author-link"]; + } + + $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `nurl` IN ('%s', '%s') AND `network` != '%s'", + intval($importer["uid"]), dbesc(normalise_link($author["author-link"])), + dbesc(normalise_link($aliaslink)), dbesc(NETWORK_STATUSNET)); + + if (dbm::is_result($r)) { + $contact = $r[0]; + $author["contact-id"] = $r[0]["id"]; + $author["author-link"] = $r[0]["url"]; + } + } elseif ($addr != "") { + // Should not happen + $contact = dba::fetch_first("SELECT * FROM `contact` WHERE `uid` = ? AND `addr` = ? AND `network` != ?", + $importer["uid"], $addr, NETWORK_STATUSNET); + + if (dbm::is_result($contact)) { + $author["contact-id"] = $contact["id"]; + $author["author-link"] = $contact["url"]; + } + } + + $avatarlist = array(); + $avatars = $xpath->query("atom:author/atom:link[@rel='avatar']", $context); + foreach ($avatars AS $avatar) { + $href = ""; + $width = 0; + foreach ($avatar->attributes AS $attributes) { + if ($attributes->name == "href") { + $href = $attributes->textContent; + } + if ($attributes->name == "width") { + $width = $attributes->textContent; + } + } + if ($href != "") { + $avatarlist[$width] = $href; + } + } + if (count($avatarlist) > 0) { + krsort($avatarlist); + $author["author-avatar"] = Probe::fixAvatar(current($avatarlist), $author["author-link"]); + } + + $displayname = $xpath->evaluate('atom:author/poco:displayName/text()', $context)->item(0)->nodeValue; + if ($displayname != "") { + $author["author-name"] = $displayname; + } + + $author["owner-name"] = $author["author-name"]; + $author["owner-link"] = $author["author-link"]; + $author["owner-avatar"] = $author["author-avatar"]; + + // Only update the contacts if it is an OStatus contact + if ($r && !$onlyfetch && ($contact["network"] == NETWORK_OSTATUS)) { + + // Update contact data + + // This query doesn't seem to work + // $value = $xpath->query("atom:link[@rel='salmon']", $context)->item(0)->nodeValue; + // if ($value != "") + // $contact["notify"] = $value; + + // This query doesn't seem to work as well - I hate these queries + // $value = $xpath->query("atom:link[@rel='self' and @type='application/atom+xml']", $context)->item(0)->nodeValue; + // if ($value != "") + // $contact["poll"] = $value; + + $value = $xpath->evaluate('atom:author/atom:uri/text()', $context)->item(0)->nodeValue; + if ($value != "") + $contact["alias"] = $value; + + $value = $xpath->evaluate('atom:author/poco:displayName/text()', $context)->item(0)->nodeValue; + if ($value != "") + $contact["name"] = $value; + + $value = $xpath->evaluate('atom:author/poco:preferredUsername/text()', $context)->item(0)->nodeValue; + if ($value != "") + $contact["nick"] = $value; + + $value = $xpath->evaluate('atom:author/poco:note/text()', $context)->item(0)->nodeValue; + if ($value != "") + $contact["about"] = html2bbcode($value); + + $value = $xpath->evaluate('atom:author/poco:address/poco:formatted/text()', $context)->item(0)->nodeValue; + if ($value != "") + $contact["location"] = $value; + + if (($contact["name"] != $r[0]["name"]) || ($contact["nick"] != $r[0]["nick"]) || ($contact["about"] != $r[0]["about"]) || + ($contact["alias"] != $r[0]["alias"]) || ($contact["location"] != $r[0]["location"])) { + + logger("Update contact data for contact ".$contact["id"], LOGGER_DEBUG); + + q("UPDATE `contact` SET `name` = '%s', `nick` = '%s', `alias` = '%s', `about` = '%s', `location` = '%s', `name-date` = '%s' WHERE `id` = %d", + dbesc($contact["name"]), dbesc($contact["nick"]), dbesc($contact["alias"]), + dbesc($contact["about"]), dbesc($contact["location"]), + dbesc(datetime_convert()), intval($contact["id"])); + } + + if (isset($author["author-avatar"]) && ($author["author-avatar"] != $r[0]['avatar'])) { + logger("Update profile picture for contact ".$contact["id"], LOGGER_DEBUG); + + update_contact_avatar($author["author-avatar"], $importer["uid"], $contact["id"]); + } + + // Ensure that we are having this contact (with uid=0) + $cid = get_contact($author["author-link"], 0); + + if ($cid) { + // Update it with the current values + q("UPDATE `contact` SET `url` = '%s', `name` = '%s', `nick` = '%s', `alias` = '%s', + `about` = '%s', `location` = '%s', + `success_update` = '%s', `last-update` = '%s' + WHERE `id` = %d", + dbesc($author["author-link"]), dbesc($contact["name"]), dbesc($contact["nick"]), + dbesc($contact["alias"]), dbesc($contact["about"]), dbesc($contact["location"]), + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($cid)); + + // Update the avatar + update_contact_avatar($author["author-avatar"], 0, $cid); + } + + $contact["generation"] = 2; + $contact["hide"] = false; // OStatus contacts are never hidden + $contact["photo"] = $author["author-avatar"]; + $gcid = update_gcontact($contact); + + link_gcontact($gcid, $contact["uid"], $contact["id"]); + } + + return $author; + } + + /** + * @brief Fetches author data from a given XML string + * + * @param string $xml The XML + * @param array $importer user record of the importing user + * + * @return array Array of author related entries for the item + */ + public static function salmon_author($xml, $importer) { + + if ($xml == "") + return; + + $doc = new DOMDocument(); + @$doc->loadXML($xml); + + $xpath = new DomXPath($doc); + $xpath->registerNamespace('atom', NAMESPACE_ATOM1); + $xpath->registerNamespace('thr', NAMESPACE_THREAD); + $xpath->registerNamespace('georss', NAMESPACE_GEORSS); + $xpath->registerNamespace('activity', NAMESPACE_ACTIVITY); + $xpath->registerNamespace('media', NAMESPACE_MEDIA); + $xpath->registerNamespace('poco', NAMESPACE_POCO); + $xpath->registerNamespace('ostatus', NAMESPACE_OSTATUS); + $xpath->registerNamespace('statusnet', NAMESPACE_STATUSNET); + + $entries = $xpath->query('/atom:entry'); + + foreach ($entries AS $entry) { + // fetch the author + $author = self::fetchauthor($xpath, $entry, $importer, $contact, true); + return $author; + } + } + + /** + * @brief Read attributes from element + * + * @param object $element Element object + * + * @return array attributes + */ + private static function read_attributes($element) { + $attribute = array(); + + foreach ($element->attributes AS $attributes) { + $attribute[$attributes->name] = $attributes->textContent; + } + + return $attribute; + } + /** * @brief Imports an XML string containing OStatus elements * @@ -543,225 +762,6 @@ class ostatus { return $link_data; } - /** - * @brief Fetches author data - * - * @param object $xpath The xpath object - * @param object $context The xml context of the author detals - * @param array $importer user record of the importing user - * @param array $contact Called by reference, will contain the fetched contact - * @param bool $onlyfetch Only fetch the header without updating the contact entries - * - * @return array Array of author related entries for the item - */ - private static function fetchauthor($xpath, $context, $importer, &$contact, $onlyfetch) { - - $author = array(); - $author["author-link"] = $xpath->evaluate('atom:author/atom:uri/text()', $context)->item(0)->nodeValue; - $author["author-name"] = $xpath->evaluate('atom:author/atom:name/text()', $context)->item(0)->nodeValue; - $addr = $xpath->evaluate('atom:author/atom:email/text()', $context)->item(0)->nodeValue; - - $aliaslink = $author["author-link"]; - - $alternate = $xpath->query("atom:author/atom:link[@rel='alternate']", $context)->item(0)->attributes; - if (is_object($alternate)) { - foreach ($alternate AS $attributes) { - if (($attributes->name == "href") && ($attributes->textContent != "")) { - $author["author-link"] = $attributes->textContent; - } - } - } - - $author["contact-id"] = $contact["id"]; - - if ($author["author-link"] != "") { - if ($aliaslink == "") { - $aliaslink = $author["author-link"]; - } - - $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `nurl` IN ('%s', '%s') AND `network` != '%s'", - intval($importer["uid"]), dbesc(normalise_link($author["author-link"])), - dbesc(normalise_link($aliaslink)), dbesc(NETWORK_STATUSNET)); - - if (dbm::is_result($r)) { - $contact = $r[0]; - $author["contact-id"] = $r[0]["id"]; - $author["author-link"] = $r[0]["url"]; - } - } elseif ($addr != "") { - // Should not happen - $contact = dba::fetch_first("SELECT * FROM `contact` WHERE `uid` = ? AND `addr` = ? AND `network` != ?", - $importer["uid"], $addr, NETWORK_STATUSNET); - - if (dbm::is_result($contact)) { - $author["contact-id"] = $contact["id"]; - $author["author-link"] = $contact["url"]; - } - } - - $avatarlist = array(); - $avatars = $xpath->query("atom:author/atom:link[@rel='avatar']", $context); - foreach ($avatars AS $avatar) { - $href = ""; - $width = 0; - foreach ($avatar->attributes AS $attributes) { - if ($attributes->name == "href") { - $href = $attributes->textContent; - } - if ($attributes->name == "width") { - $width = $attributes->textContent; - } - } - if ($href != "") { - $avatarlist[$width] = $href; - } - } - if (count($avatarlist) > 0) { - krsort($avatarlist); - $author["author-avatar"] = Probe::fixAvatar(current($avatarlist), $author["author-link"]); - } - - $displayname = $xpath->evaluate('atom:author/poco:displayName/text()', $context)->item(0)->nodeValue; - if ($displayname != "") { - $author["author-name"] = $displayname; - } - - $author["owner-name"] = $author["author-name"]; - $author["owner-link"] = $author["author-link"]; - $author["owner-avatar"] = $author["author-avatar"]; - - // Only update the contacts if it is an OStatus contact - if ($r && !$onlyfetch && ($contact["network"] == NETWORK_OSTATUS)) { - - // Update contact data - - // This query doesn't seem to work - // $value = $xpath->query("atom:link[@rel='salmon']", $context)->item(0)->nodeValue; - // if ($value != "") - // $contact["notify"] = $value; - - // This query doesn't seem to work as well - I hate these queries - // $value = $xpath->query("atom:link[@rel='self' and @type='application/atom+xml']", $context)->item(0)->nodeValue; - // if ($value != "") - // $contact["poll"] = $value; - - $value = $xpath->evaluate('atom:author/atom:uri/text()', $context)->item(0)->nodeValue; - if ($value != "") - $contact["alias"] = $value; - - $value = $xpath->evaluate('atom:author/poco:displayName/text()', $context)->item(0)->nodeValue; - if ($value != "") - $contact["name"] = $value; - - $value = $xpath->evaluate('atom:author/poco:preferredUsername/text()', $context)->item(0)->nodeValue; - if ($value != "") - $contact["nick"] = $value; - - $value = $xpath->evaluate('atom:author/poco:note/text()', $context)->item(0)->nodeValue; - if ($value != "") - $contact["about"] = html2bbcode($value); - - $value = $xpath->evaluate('atom:author/poco:address/poco:formatted/text()', $context)->item(0)->nodeValue; - if ($value != "") - $contact["location"] = $value; - - if (($contact["name"] != $r[0]["name"]) || ($contact["nick"] != $r[0]["nick"]) || ($contact["about"] != $r[0]["about"]) || - ($contact["alias"] != $r[0]["alias"]) || ($contact["location"] != $r[0]["location"])) { - - logger("Update contact data for contact ".$contact["id"], LOGGER_DEBUG); - - q("UPDATE `contact` SET `name` = '%s', `nick` = '%s', `alias` = '%s', `about` = '%s', `location` = '%s', `name-date` = '%s' WHERE `id` = %d", - dbesc($contact["name"]), dbesc($contact["nick"]), dbesc($contact["alias"]), - dbesc($contact["about"]), dbesc($contact["location"]), - dbesc(datetime_convert()), intval($contact["id"])); - } - - if (isset($author["author-avatar"]) && ($author["author-avatar"] != $r[0]['avatar'])) { - logger("Update profile picture for contact ".$contact["id"], LOGGER_DEBUG); - - update_contact_avatar($author["author-avatar"], $importer["uid"], $contact["id"]); - } - - // Ensure that we are having this contact (with uid=0) - $cid = get_contact($author["author-link"], 0); - - if ($cid) { - // Update it with the current values - q("UPDATE `contact` SET `url` = '%s', `name` = '%s', `nick` = '%s', `alias` = '%s', - `about` = '%s', `location` = '%s', - `success_update` = '%s', `last-update` = '%s' - WHERE `id` = %d", - dbesc($author["author-link"]), dbesc($contact["name"]), dbesc($contact["nick"]), - dbesc($contact["alias"]), dbesc($contact["about"]), dbesc($contact["location"]), - dbesc(datetime_convert()), dbesc(datetime_convert()), intval($cid)); - - // Update the avatar - update_contact_avatar($author["author-avatar"], 0, $cid); - } - - $contact["generation"] = 2; - $contact["hide"] = false; // OStatus contacts are never hidden - $contact["photo"] = $author["author-avatar"]; - $gcid = update_gcontact($contact); - - link_gcontact($gcid, $contact["uid"], $contact["id"]); - } - - return $author; - } - - /** - * @brief Fetches author data from a given XML string - * - * @param string $xml The XML - * @param array $importer user record of the importing user - * - * @return array Array of author related entries for the item - */ - public static function salmon_author($xml, $importer) { - - if ($xml == "") - return; - - $doc = new DOMDocument(); - @$doc->loadXML($xml); - - $xpath = new DomXPath($doc); - $xpath->registerNamespace('atom', NAMESPACE_ATOM1); - $xpath->registerNamespace('thr', NAMESPACE_THREAD); - $xpath->registerNamespace('georss', NAMESPACE_GEORSS); - $xpath->registerNamespace('activity', NAMESPACE_ACTIVITY); - $xpath->registerNamespace('media', NAMESPACE_MEDIA); - $xpath->registerNamespace('poco', NAMESPACE_POCO); - $xpath->registerNamespace('ostatus', NAMESPACE_OSTATUS); - $xpath->registerNamespace('statusnet', NAMESPACE_STATUSNET); - - $entries = $xpath->query('/atom:entry'); - - foreach ($entries AS $entry) { - // fetch the author - $author = self::fetchauthor($xpath, $entry, $importer, $contact, true); - return $author; - } - } - - /** - * @brief Read attributes from element - * - * @param object $element Element object - * - * @return array attributes - */ - private static function read_attributes($element) { - $attribute = array(); - - foreach ($element->attributes AS $attributes) { - $attribute[$attributes->name] = $attributes->textContent; - } - - return $attribute; - } - /** * @brief Checks if the current post is a reshare * From 21a630ebae9e1cfdac8c5148318f719ec89fbd73 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 10 Sep 2017 07:52:07 +0000 Subject: [PATCH 4/9] Added documentation --- include/ostatus.php | 52 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/include/ostatus.php b/include/ostatus.php index d05e575f0..c13a466ed 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -29,9 +29,6 @@ require_once 'include/cache.php'; * */ class ostatus { - const OSTATUS_DEFAULT_POLL_INTERVAL = 30; // given in minutes - const OSTATUS_DEFAULT_POLL_TIMEFRAME = 1440; // given in minutes - const OSTATUS_DEFAULT_POLL_TIMEFRAME_MENTIONS = 14400; // given in minutes private static $itemlist; @@ -39,7 +36,7 @@ class ostatus { * @brief Fetches author data * * @param object $xpath The xpath object - * @param object $context The xml context of the author detals + * @param object $context The xml context of the author details * @param array $importer user record of the importing user * @param array $contact Called by reference, will contain the fetched contact * @param bool $onlyfetch Only fetch the header without updating the contact entries @@ -259,20 +256,24 @@ class ostatus { * * @param string $xml The XML * @param array $importer user record of the importing user - * @param $contact - * @param array $hub Called by reference, returns the fetched hub data + * @param array $contact + * @param string $hub Called by reference, returns the fetched hub data */ public static function import($xml, $importer, &$contact, &$hub) { self::process($xml, $importer, $contact, $hub); } /** - * @brief Imports an XML string containing OStatus elements + * @brief Internal feed processing * * @param string $xml The XML * @param array $importer user record of the importing user - * @param $contact - * @param array $hub Called by reference, returns the fetched hub data + * @param array $contact + * @param string $hub Called by reference, returns the fetched hub data + * @param boolean $stored Is the post fresh imported or from the database? + * @param boolean $initialize Is it the leading post so that data has to be initialized? + * + * @return boolean Could the XML be processed? */ private static function process($xml, $importer, &$contact, &$hub, $stored = false, $initialize = true) { if ($initialize) { @@ -465,6 +466,14 @@ class ostatus { return true; } + /** + * @brief Processes the XML for a post + * + * @param object $xpath The xpath object + * @param object $entry The xml entry that is processed + * @param array $item The item array + * @param array $importer user record of the importing user + */ private static function processPost($xpath, $entry, &$item, $importer) { $item["uri"] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue; $item["body"] = html2bbcode($xpath->query('atom:content/text()', $entry)->item(0)->nodeValue); @@ -591,6 +600,13 @@ class ostatus { self::$itemlist[] = $item; } + /** + * @brief Fetch related posts and processes them + * + * @param string $related The link to the related item + * @param string $related_uri The related item in "uri" format + * @param array $importer user record of the importing user + */ private static function fetchRelated($related, $related_uri, $importer) { $condition = array('`item-uri` = ? AND `protocol` IN (?, ?)', $related_uri, PROTOCOL_DFRN, PROTOCOL_OSTATUS_SALMON); $conversation = dba::select('conversation', array('source', 'protocol'), $condition, array('limit' => 1)); @@ -655,6 +671,16 @@ class ostatus { return; } + /** + * @brief Processes the XML for a repeated post + * + * @param object $xpath The xpath object + * @param object $entry The xml entry that is processed + * @param array $item The item array + * @param array $importer user record of the importing user + * + * @return array with data from links + */ private static function processRepeatedItem($xpath, $entry, &$item, $importer) { $activityobjects = $xpath->query('activity:object', $entry)->item(0); @@ -704,6 +730,14 @@ class ostatus { return $link_data; } + /** + * @brief Processes links in the XML + * + * @param object $links The xml data that contain links + * @param array $item The item array + * + * @return array with data from the links + */ private static function processLinks($links, &$item) { $link_data = array('add_body' => '', 'self' => ''); From bdf490e2fad7415c70df8dd2d57473cee3055a48 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 10 Sep 2017 07:58:14 +0000 Subject: [PATCH 5/9] Added missing function --- include/items.php | 1 + include/ostatus.php | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/include/items.php b/include/items.php index ec4c986a1..f2ee2311b 100644 --- a/include/items.php +++ b/include/items.php @@ -534,6 +534,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f } // Converting the plink + /// @todo Check if this is really still needed if ($arr['network'] == NETWORK_OSTATUS) { if (isset($arr['plink'])) { $arr['plink'] = ostatus::convert_href($arr['plink']); diff --git a/include/ostatus.php b/include/ostatus.php index c13a466ed..02bd2f447 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -796,6 +796,36 @@ class ostatus { return $link_data; } +/** + * @brief Create an url out of an uri + * + * @param string $href URI in the format "parameter1:parameter1:..." + * + * @return string URL in the format http(s)://.... + */ + public static function convert_href($href) { + $elements = explode(":",$href); + + if ((count($elements) <= 2) || ($elements[0] != "tag")) + return $href; + + $server = explode(",", $elements[1]); + $conversation = explode("=", $elements[2]); + + if ((count($elements) == 4) && ($elements[2] == "post")) + return "http://".$server[0]."/notice/".$elements[3]; + + if ((count($conversation) != 2) || ($conversation[1] =="")) { + return $href; + } + if ($elements[3] == "objectType=thread") { + return "http://".$server[0]."/conversation/".$conversation[1]; + } else { + return "http://".$server[0]."/notice/".$conversation[1]; + } + return $href; + } + /** * @brief Checks if the current post is a reshare * From b847f63e2ac8005f2d2696251860fe50e3e61bc2 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 10 Sep 2017 08:27:24 +0000 Subject: [PATCH 6/9] Added logging --- include/ostatus.php | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/include/ostatus.php b/include/ostatus.php index 02bd2f447..6fb42c8f0 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -614,9 +614,11 @@ class ostatus { $stored = true; $xml = $conversation['source']; if (self::process($xml, $importer, $contact, $hub, $stored, false)) { + logger('Got valid cached XML for URI '.$related_uri, LOGGER_DEBUG); return; } if ($conversation['protocol'] == PROTOCOL_OSTATUS_SALMON) { + logger('Delete invalid cached XML for URI '.$related_uri, LOGGER_DEBUG); dba::delete('conversation', array('item-uri' => $related_uri)); } } @@ -631,6 +633,7 @@ class ostatus { $xml = ''; if (stristr($related_data['header'], 'Content-Type: application/atom+xml')) { + logger('Directly fetched XML for URI '.$related_uri, LOGGER_DEBUG); $xml = $related_data['body']; } @@ -641,16 +644,22 @@ class ostatus { } $xpath = new DomXPath($doc); + $atom_file = ''; + $links = $xpath->query('//link'); if ($links) { foreach ($links AS $link) { $attribute = self::read_attributes($link); if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) { - $related_atom = z_fetch_url($attribute['href']); + $atom_file = $attribute['href']; + } + } + if ($atom_file != '') { + $related_atom = z_fetch_url($atom_file); - if ($related_atom['success']) { - $xml = $related_atom['body']; - } + if ($related_atom['success']) { + logger('Fetched XML for URI '.$related_uri, LOGGER_DEBUG); + $xml = $related_atom['body']; } } } @@ -658,15 +667,29 @@ class ostatus { // Workaround for older GNU Social servers if (($xml == '') && strstr($related, '/notice/')) { - $related_atom = z_fetch_url(str_replace('/notice/', '/api/statuses/show/', $related).',atom'); + $related_atom = z_fetch_url(str_replace('/notice/', '/api/statuses/show/', $related).'.atom'); if ($related_atom['success']) { + logger('GNU Social workaround to fetch XML for URI '.$related_uri, LOGGER_DEBUG); + $xml = $related_atom['body']; + } + } + + // Even more worse workaround for GNU Social ;-) + if ($xml == '') { + $related_guess = ostatus::convert_href($related_uri); + $related_atom = z_fetch_url(str_replace('/notice/', '/api/statuses/show/', $related_guess).'.atom'); + + if ($related_atom['success']) { + logger('GNU Social workaround 2 to fetch XML for URI '.$related_uri, LOGGER_DEBUG); $xml = $related_atom['body']; } } if ($xml != '') { self::process($xml, $importer, $contact, $hub, $stored, false); + } else { + logger("XML couldn't be fetched for URI: ".$related_uri." - href: ".$related, LOGGER_DEBUG); } return; } From 8e17be7225cd0032949724451e792f4f9f65dfb3 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 10 Sep 2017 21:56:05 +0000 Subject: [PATCH 7/9] Store conversation data --- boot.php | 5 +- include/ostatus.php | 173 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 161 insertions(+), 17 deletions(-) diff --git a/boot.php b/boot.php index b8be70229..417502b96 100644 --- a/boot.php +++ b/boot.php @@ -229,8 +229,9 @@ define('PROTOCOL_UNKNOWN', 0); define('PROTOCOL_DFRN', 1); define('PROTOCOL_DIASPORA', 2); define('PROTOCOL_OSTATUS_SALMON', 3); -define('PROTOCOL_OSTATUS_FEED', 4); -define('PROTOCOL_GS_CONVERSATION', 5); +define('PROTOCOL_OSTATUS_FEED', 4); // Deprecated +define('PROTOCOL_GS_CONVERSATION', 5); // Deprecated +define('PROTOCOL_SPLITTED_CONV', 6); /** @}*/ /** diff --git a/include/ostatus.php b/include/ostatus.php index 6fb42c8f0..d33073a1a 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -31,6 +31,7 @@ require_once 'include/cache.php'; class ostatus { private static $itemlist; + private static $conv_list = array(); /** * @brief Fetches author data @@ -278,6 +279,7 @@ class ostatus { private static function process($xml, $importer, &$contact, &$hub, $stored = false, $initialize = true) { if ($initialize) { self::$itemlist = array(); + self::$conv_list = array(); } logger("Import OStatus message", LOGGER_DEBUG); @@ -321,10 +323,19 @@ class ostatus { if ($first_child == "feed") { $entries = $xpath->query('/atom:feed/atom:entry'); - $header["protocol"] = PROTOCOL_OSTATUS_FEED; } else { $entries = $xpath->query('/atom:entry'); + } + + if ($entries->length == 1) { + $doc2 = new DOMDocument(); + $doc2->loadXML($xml); + $doc2->preserveWhiteSpace = false; + $doc2->formatOutput = true; + $xml2 = $doc2->saveXML(); + $header["protocol"] = PROTOCOL_OSTATUS_SALMON; + $header["source"] = $xml2; } // Fetch the first author @@ -332,7 +343,6 @@ class ostatus { $author = self::fetchauthor($xpath, $authordata, $importer, $contact, $stored); $entry = $xpath->query('/atom:entry'); - $header["protocol"] = PROTOCOL_OSTATUS_SALMON; // Reverse the order of the entries $entrylist = array(); @@ -406,14 +416,6 @@ class ostatus { logger("Unhandled verb ".$item["verb"]." ".print_r($item, true)); } - $doc2 = new DOMDocument(); - $doc2->loadXML($xml); - $doc2->preserveWhiteSpace = false; - $doc2->formatOutput = true; - $xml2 = $doc2->saveXML(); - - $item["source"] = $xml2; - self::processPost($xpath, $entry, $item, $importer); if ($initialize && (count(self::$itemlist) > 0)) { @@ -502,10 +504,6 @@ class ostatus { } } - if (empty($item['conversation-href']) && !empty($item['conversation-uri'])) { - $item['conversation-href'] = $item['conversation-uri']; - } - $related = ""; $inreplyto = $xpath->query('thr:in-reply-to', $entry); @@ -585,6 +583,10 @@ class ostatus { $item["body"] = html2bbcode($clear_text) . '[spoiler]' . $item["body"] . '[/spoiler]'; } + if (!empty($item["conversation-href"])) { + self::fetchConversation($item['conversation-href'], $item['conversation-uri']); + } + if (isset($item["parent-uri"]) && ($related != '')) { self::FetchRelated($related, $item["parent-uri"], $importer); $item["type"] = 'remote-comment'; @@ -593,13 +595,143 @@ class ostatus { $item["parent-uri"] = $item["uri"]; } - if ($item['author-link'] != '') { + if (($item['author-link'] != '') && !empty($header["protocol"])) { $item = store_conversation($item); } self::$itemlist[] = $item; } + /** + * @brief Fetch the conversation for posts + * + * @param string $conversation The link to the conversation + * @param string $conversation_uri The conversation in "uri" format + */ + private static function fetchConversation($conversation, $conversation_uri) { + + // Ensure that we only store a conversation once in a process + if (isset(self::$conv_list[$conversation])) { + return; + } + + self::$conv_list[$conversation] = true; + + $conversation_data = z_fetch_url($conversation); + + if (!$conversation_data['success']) { + return; + } + + $xml = ''; + + if (stristr($conversation_data['header'], 'Content-Type: application/atom+xml')) { + $xml = $conversation_data['body']; + } + + if ($xml == '') { + $doc = new DOMDocument(); + if (!@$doc->loadHTML($conversation_data['body'])) { + return; + } + $xpath = new DomXPath($doc); + + $links = $xpath->query('//link'); + if ($links) { + foreach ($links AS $link) { + $attribute = ostatus::read_attributes($link); + if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) { + $file = $attribute['href']; + } + } + if ($file != '') { + $conversation_atom = z_fetch_url($attribute['href']); + + if ($conversation_atom['success']) { + $xml = $conversation_atom['body']; + } + } + } + } + + if ($xml == '') { + return; + } + + self::storeConversation($xml, $conversation, $conversation_uri); + } + + /** + * @brief Store a feed in several conversation entries + * + * @param string $xml The feed + */ + private static function storeConversation($xml, $conversation = '', $conversation_uri = '') { + $doc = new DOMDocument(); + @$doc->loadXML($xml); + + $xpath = new DomXPath($doc); + $xpath->registerNamespace('atom', NAMESPACE_ATOM1); + $xpath->registerNamespace('thr', NAMESPACE_THREAD); + $xpath->registerNamespace('ostatus', NAMESPACE_OSTATUS); + + $entries = $xpath->query('/atom:feed/atom:entry'); + + // Now store the entries + foreach ($entries AS $entry) { + $doc2 = new DOMDocument(); + $doc2->preserveWhiteSpace = false; + $doc2->formatOutput = true; + + $conv_data = array(); + + $conv_data['protocol'] = PROTOCOL_SPLITTED_CONV; + $conv_data['network'] = NETWORK_OSTATUS; + $conv_data['uri'] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue; + + $inreplyto = $xpath->query('thr:in-reply-to', $entry); + if (is_object($inreplyto->item(0))) { + foreach ($inreplyto->item(0)->attributes AS $attributes) { + if ($attributes->name == "ref") { + $conv_data['reply-to-uri'] = $attributes->textContent; + } + } + } + + $conv = $xpath->query('ostatus:conversation/text()', $entry)->item(0)->nodeValue; + $conv_data['conversation-uri'] = $conv; + + $conv = $xpath->query('ostatus:conversation', $entry); + if (is_object($conv->item(0))) { + foreach ($conv->item(0)->attributes AS $attributes) { + if ($attributes->name == "ref") { + $conv_data['conversation-uri'] = $attributes->textContent; + } + if ($attributes->name == "href") { + $conv_data['conversation-href'] = $attributes->textContent; + } + } + } + + if ($conversation != '') { + $conv_data['conversation-uri'] = $conversation; + } + + if ($conversation_uri != '') { + $conv_data['conversation-uri'] = $conversation_uri; + } + + $entry = $doc2->importNode($entry, true); + + $doc2->appendChild($entry); + + $conv_data['source'] = $doc2->saveXML(); + + logger('Store conversation data for uri '.$conv_data['uri'], LOGGER_DEBUG); + store_conversation($conv_data); + } + } + /** * @brief Fetch related posts and processes them * @@ -686,6 +818,17 @@ class ostatus { } } + // Finally we take the data that we fetched from "ostatus:conversation" + if ($xml == '') { + $condition = array('item-uri' => $related_uri, 'protocol' => PROTOCOL_SPLITTED_CONV); + $conversation = dba::select('conversation', array('source'), $condition, array('limit' => 1)); + if (dbm::is_result($conversation)) { + $stored = true; + logger('Got cached XML from conversation for URI '.$related_uri, LOGGER_DEBUG); + $xml = $conversation['source']; + } + } + if ($xml != '') { self::process($xml, $importer, $contact, $hub, $stored, false); } else { From ec929d8207e519837c73601f7193c334e0337902 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 11 Sep 2017 12:44:37 +0000 Subject: [PATCH 8/9] Fetch the own single post if we are processing a feed of multiple entries --- include/ostatus.php | 57 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/include/ostatus.php b/include/ostatus.php index d33073a1a..0c99905fa 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -328,6 +328,7 @@ class ostatus { } if ($entries->length == 1) { + // We reformat the XML to make it better readable $doc2 = new DOMDocument(); $doc2->loadXML($xml); $doc2->preserveWhiteSpace = false; @@ -336,6 +337,8 @@ class ostatus { $header["protocol"] = PROTOCOL_OSTATUS_SALMON; $header["source"] = $xml2; + } elseif (!$initialize) { + return false; } // Fetch the first author @@ -351,10 +354,6 @@ class ostatus { $entrylist[] = $entry; } - if (!$initialize && (count($entrylist) > 1)) { - return false; - } - foreach (array_reverse($entrylist) AS $entry) { // fetch the author $authorelement = $xpath->query('/atom:entry/atom:author', $entry); @@ -583,19 +582,23 @@ class ostatus { $item["body"] = html2bbcode($clear_text) . '[spoiler]' . $item["body"] . '[/spoiler]'; } + if (($self != '') && empty($item['protocol'])) { + self::fetchSelf($self, $item); + } + if (!empty($item["conversation-href"])) { self::fetchConversation($item['conversation-href'], $item['conversation-uri']); } if (isset($item["parent-uri"]) && ($related != '')) { - self::FetchRelated($related, $item["parent-uri"], $importer); + self::fetchRelated($related, $item["parent-uri"], $importer); $item["type"] = 'remote-comment'; $item["gravity"] = GRAVITY_COMMENT; } else { $item["parent-uri"] = $item["uri"]; } - if (($item['author-link'] != '') && !empty($header["protocol"])) { + if (($item['author-link'] != '') && !empty($item['protocol'])) { $item = store_conversation($item); } @@ -727,11 +730,53 @@ class ostatus { $conv_data['source'] = $doc2->saveXML(); + $condition = array('item-uri' => $conv_data['uri'],'protocol' => PROTOCOL_OSTATUS_FEED); + if (dba::exists('conversation', $condition)) { + logger('Delete deprecated entry for URI '.$conv_data['uri'], LOGGER_DEBUG); + dba::delete('conversation', array('item-uri' => $conv_data['uri'])); + } + logger('Store conversation data for uri '.$conv_data['uri'], LOGGER_DEBUG); store_conversation($conv_data); } } + /** + * @brief Fetch the own post so that it can be stored later + * @param array $item The item array + * + * We want to store the original data for later processing. + * This function is meant for cases where we process a feed with multiple entries. + * In that case we need to fetch the single posts here. + * + * @param string $self The link to the self item + */ + private static function fetchSelf($self, &$item) { + $condition = array('`item-uri` = ? AND `protocol` IN (?, ?)', $self, PROTOCOL_DFRN, PROTOCOL_OSTATUS_SALMON); + if (dba::exists('conversation', $condition)) { + logger('Conversation '.$item['uri'].' is already stored.'); + return; + } + + $self_data = z_fetch_url($self); + + if (!$self_data['success']) { + return; + } + + // We reformat the XML to make it better readable + $doc = new DOMDocument(); + $doc->loadXML($self_data['body']); + $doc->preserveWhiteSpace = false; + $doc->formatOutput = true; + $xml = $doc->saveXML(); + + $item["protocol"] = PROTOCOL_OSTATUS_SALMON; + $item["source"] = $xml; + + logger('Conversation '.$item['uri'].' is now fetched.'); + } + /** * @brief Fetch related posts and processes them * From 519a8807f1bac539b9efa433550e3a6c1d0d6a97 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 11 Sep 2017 22:15:16 +0000 Subject: [PATCH 9/9] Added documentation --- include/dfrn.php | 1 + 1 file changed, 1 insertion(+) diff --git a/include/dfrn.php b/include/dfrn.php index 0b12f4d61..c8e89fe40 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -820,6 +820,7 @@ class dfrn { * @param array $owner Owner record * @param bool $comment Trigger the sending of the "comment" element * @param int $cid Contact ID of the recipient + * @param bool $single If set, the entry is created as an XML document with a single "entry" element * * @return object XML entry object * @todo Add type-hints