From 8e17be7225cd0032949724451e792f4f9f65dfb3 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 10 Sep 2017 21:56:05 +0000 Subject: [PATCH] Store conversation data --- boot.php | 5 +- include/ostatus.php | 173 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 161 insertions(+), 17 deletions(-) diff --git a/boot.php b/boot.php index b8be70229c..417502b960 100644 --- a/boot.php +++ b/boot.php @@ -229,8 +229,9 @@ define('PROTOCOL_UNKNOWN', 0); define('PROTOCOL_DFRN', 1); define('PROTOCOL_DIASPORA', 2); define('PROTOCOL_OSTATUS_SALMON', 3); -define('PROTOCOL_OSTATUS_FEED', 4); -define('PROTOCOL_GS_CONVERSATION', 5); +define('PROTOCOL_OSTATUS_FEED', 4); // Deprecated +define('PROTOCOL_GS_CONVERSATION', 5); // Deprecated +define('PROTOCOL_SPLITTED_CONV', 6); /** @}*/ /** diff --git a/include/ostatus.php b/include/ostatus.php index 6fb42c8f0a..d33073a1a7 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -31,6 +31,7 @@ require_once 'include/cache.php'; class ostatus { private static $itemlist; + private static $conv_list = array(); /** * @brief Fetches author data @@ -278,6 +279,7 @@ class ostatus { private static function process($xml, $importer, &$contact, &$hub, $stored = false, $initialize = true) { if ($initialize) { self::$itemlist = array(); + self::$conv_list = array(); } logger("Import OStatus message", LOGGER_DEBUG); @@ -321,10 +323,19 @@ class ostatus { if ($first_child == "feed") { $entries = $xpath->query('/atom:feed/atom:entry'); - $header["protocol"] = PROTOCOL_OSTATUS_FEED; } else { $entries = $xpath->query('/atom:entry'); + } + + if ($entries->length == 1) { + $doc2 = new DOMDocument(); + $doc2->loadXML($xml); + $doc2->preserveWhiteSpace = false; + $doc2->formatOutput = true; + $xml2 = $doc2->saveXML(); + $header["protocol"] = PROTOCOL_OSTATUS_SALMON; + $header["source"] = $xml2; } // Fetch the first author @@ -332,7 +343,6 @@ class ostatus { $author = self::fetchauthor($xpath, $authordata, $importer, $contact, $stored); $entry = $xpath->query('/atom:entry'); - $header["protocol"] = PROTOCOL_OSTATUS_SALMON; // Reverse the order of the entries $entrylist = array(); @@ -406,14 +416,6 @@ class ostatus { logger("Unhandled verb ".$item["verb"]." ".print_r($item, true)); } - $doc2 = new DOMDocument(); - $doc2->loadXML($xml); - $doc2->preserveWhiteSpace = false; - $doc2->formatOutput = true; - $xml2 = $doc2->saveXML(); - - $item["source"] = $xml2; - self::processPost($xpath, $entry, $item, $importer); if ($initialize && (count(self::$itemlist) > 0)) { @@ -502,10 +504,6 @@ class ostatus { } } - if (empty($item['conversation-href']) && !empty($item['conversation-uri'])) { - $item['conversation-href'] = $item['conversation-uri']; - } - $related = ""; $inreplyto = $xpath->query('thr:in-reply-to', $entry); @@ -585,6 +583,10 @@ class ostatus { $item["body"] = html2bbcode($clear_text) . '[spoiler]' . $item["body"] . '[/spoiler]'; } + if (!empty($item["conversation-href"])) { + self::fetchConversation($item['conversation-href'], $item['conversation-uri']); + } + if (isset($item["parent-uri"]) && ($related != '')) { self::FetchRelated($related, $item["parent-uri"], $importer); $item["type"] = 'remote-comment'; @@ -593,13 +595,143 @@ class ostatus { $item["parent-uri"] = $item["uri"]; } - if ($item['author-link'] != '') { + if (($item['author-link'] != '') && !empty($header["protocol"])) { $item = store_conversation($item); } self::$itemlist[] = $item; } + /** + * @brief Fetch the conversation for posts + * + * @param string $conversation The link to the conversation + * @param string $conversation_uri The conversation in "uri" format + */ + private static function fetchConversation($conversation, $conversation_uri) { + + // Ensure that we only store a conversation once in a process + if (isset(self::$conv_list[$conversation])) { + return; + } + + self::$conv_list[$conversation] = true; + + $conversation_data = z_fetch_url($conversation); + + if (!$conversation_data['success']) { + return; + } + + $xml = ''; + + if (stristr($conversation_data['header'], 'Content-Type: application/atom+xml')) { + $xml = $conversation_data['body']; + } + + if ($xml == '') { + $doc = new DOMDocument(); + if (!@$doc->loadHTML($conversation_data['body'])) { + return; + } + $xpath = new DomXPath($doc); + + $links = $xpath->query('//link'); + if ($links) { + foreach ($links AS $link) { + $attribute = ostatus::read_attributes($link); + if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) { + $file = $attribute['href']; + } + } + if ($file != '') { + $conversation_atom = z_fetch_url($attribute['href']); + + if ($conversation_atom['success']) { + $xml = $conversation_atom['body']; + } + } + } + } + + if ($xml == '') { + return; + } + + self::storeConversation($xml, $conversation, $conversation_uri); + } + + /** + * @brief Store a feed in several conversation entries + * + * @param string $xml The feed + */ + private static function storeConversation($xml, $conversation = '', $conversation_uri = '') { + $doc = new DOMDocument(); + @$doc->loadXML($xml); + + $xpath = new DomXPath($doc); + $xpath->registerNamespace('atom', NAMESPACE_ATOM1); + $xpath->registerNamespace('thr', NAMESPACE_THREAD); + $xpath->registerNamespace('ostatus', NAMESPACE_OSTATUS); + + $entries = $xpath->query('/atom:feed/atom:entry'); + + // Now store the entries + foreach ($entries AS $entry) { + $doc2 = new DOMDocument(); + $doc2->preserveWhiteSpace = false; + $doc2->formatOutput = true; + + $conv_data = array(); + + $conv_data['protocol'] = PROTOCOL_SPLITTED_CONV; + $conv_data['network'] = NETWORK_OSTATUS; + $conv_data['uri'] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue; + + $inreplyto = $xpath->query('thr:in-reply-to', $entry); + if (is_object($inreplyto->item(0))) { + foreach ($inreplyto->item(0)->attributes AS $attributes) { + if ($attributes->name == "ref") { + $conv_data['reply-to-uri'] = $attributes->textContent; + } + } + } + + $conv = $xpath->query('ostatus:conversation/text()', $entry)->item(0)->nodeValue; + $conv_data['conversation-uri'] = $conv; + + $conv = $xpath->query('ostatus:conversation', $entry); + if (is_object($conv->item(0))) { + foreach ($conv->item(0)->attributes AS $attributes) { + if ($attributes->name == "ref") { + $conv_data['conversation-uri'] = $attributes->textContent; + } + if ($attributes->name == "href") { + $conv_data['conversation-href'] = $attributes->textContent; + } + } + } + + if ($conversation != '') { + $conv_data['conversation-uri'] = $conversation; + } + + if ($conversation_uri != '') { + $conv_data['conversation-uri'] = $conversation_uri; + } + + $entry = $doc2->importNode($entry, true); + + $doc2->appendChild($entry); + + $conv_data['source'] = $doc2->saveXML(); + + logger('Store conversation data for uri '.$conv_data['uri'], LOGGER_DEBUG); + store_conversation($conv_data); + } + } + /** * @brief Fetch related posts and processes them * @@ -686,6 +818,17 @@ class ostatus { } } + // Finally we take the data that we fetched from "ostatus:conversation" + if ($xml == '') { + $condition = array('item-uri' => $related_uri, 'protocol' => PROTOCOL_SPLITTED_CONV); + $conversation = dba::select('conversation', array('source'), $condition, array('limit' => 1)); + if (dbm::is_result($conversation)) { + $stored = true; + logger('Got cached XML from conversation for URI '.$related_uri, LOGGER_DEBUG); + $xml = $conversation['source']; + } + } + if ($xml != '') { self::process($xml, $importer, $contact, $hub, $stored, false); } else {