Store conversation data

This commit is contained in:
Michael 2017-09-10 21:56:05 +00:00
parent b847f63e2a
commit 8e17be7225
2 changed files with 161 additions and 17 deletions

View file

@ -229,8 +229,9 @@ define('PROTOCOL_UNKNOWN', 0);
define('PROTOCOL_DFRN', 1); define('PROTOCOL_DFRN', 1);
define('PROTOCOL_DIASPORA', 2); define('PROTOCOL_DIASPORA', 2);
define('PROTOCOL_OSTATUS_SALMON', 3); define('PROTOCOL_OSTATUS_SALMON', 3);
define('PROTOCOL_OSTATUS_FEED', 4); define('PROTOCOL_OSTATUS_FEED', 4); // Deprecated
define('PROTOCOL_GS_CONVERSATION', 5); define('PROTOCOL_GS_CONVERSATION', 5); // Deprecated
define('PROTOCOL_SPLITTED_CONV', 6);
/** @}*/ /** @}*/
/** /**

View file

@ -31,6 +31,7 @@ require_once 'include/cache.php';
class ostatus { class ostatus {
private static $itemlist; private static $itemlist;
private static $conv_list = array();
/** /**
* @brief Fetches author data * @brief Fetches author data
@ -278,6 +279,7 @@ class ostatus {
private static function process($xml, $importer, &$contact, &$hub, $stored = false, $initialize = true) { private static function process($xml, $importer, &$contact, &$hub, $stored = false, $initialize = true) {
if ($initialize) { if ($initialize) {
self::$itemlist = array(); self::$itemlist = array();
self::$conv_list = array();
} }
logger("Import OStatus message", LOGGER_DEBUG); logger("Import OStatus message", LOGGER_DEBUG);
@ -321,10 +323,19 @@ class ostatus {
if ($first_child == "feed") { if ($first_child == "feed") {
$entries = $xpath->query('/atom:feed/atom:entry'); $entries = $xpath->query('/atom:feed/atom:entry');
$header["protocol"] = PROTOCOL_OSTATUS_FEED;
} else { } else {
$entries = $xpath->query('/atom:entry'); $entries = $xpath->query('/atom:entry');
}
if ($entries->length == 1) {
$doc2 = new DOMDocument();
$doc2->loadXML($xml);
$doc2->preserveWhiteSpace = false;
$doc2->formatOutput = true;
$xml2 = $doc2->saveXML();
$header["protocol"] = PROTOCOL_OSTATUS_SALMON; $header["protocol"] = PROTOCOL_OSTATUS_SALMON;
$header["source"] = $xml2;
} }
// Fetch the first author // Fetch the first author
@ -332,7 +343,6 @@ class ostatus {
$author = self::fetchauthor($xpath, $authordata, $importer, $contact, $stored); $author = self::fetchauthor($xpath, $authordata, $importer, $contact, $stored);
$entry = $xpath->query('/atom:entry'); $entry = $xpath->query('/atom:entry');
$header["protocol"] = PROTOCOL_OSTATUS_SALMON;
// Reverse the order of the entries // Reverse the order of the entries
$entrylist = array(); $entrylist = array();
@ -406,14 +416,6 @@ class ostatus {
logger("Unhandled verb ".$item["verb"]." ".print_r($item, true)); logger("Unhandled verb ".$item["verb"]." ".print_r($item, true));
} }
$doc2 = new DOMDocument();
$doc2->loadXML($xml);
$doc2->preserveWhiteSpace = false;
$doc2->formatOutput = true;
$xml2 = $doc2->saveXML();
$item["source"] = $xml2;
self::processPost($xpath, $entry, $item, $importer); self::processPost($xpath, $entry, $item, $importer);
if ($initialize && (count(self::$itemlist) > 0)) { if ($initialize && (count(self::$itemlist) > 0)) {
@ -502,10 +504,6 @@ class ostatus {
} }
} }
if (empty($item['conversation-href']) && !empty($item['conversation-uri'])) {
$item['conversation-href'] = $item['conversation-uri'];
}
$related = ""; $related = "";
$inreplyto = $xpath->query('thr:in-reply-to', $entry); $inreplyto = $xpath->query('thr:in-reply-to', $entry);
@ -585,6 +583,10 @@ class ostatus {
$item["body"] = html2bbcode($clear_text) . '[spoiler]' . $item["body"] . '[/spoiler]'; $item["body"] = html2bbcode($clear_text) . '[spoiler]' . $item["body"] . '[/spoiler]';
} }
if (!empty($item["conversation-href"])) {
self::fetchConversation($item['conversation-href'], $item['conversation-uri']);
}
if (isset($item["parent-uri"]) && ($related != '')) { if (isset($item["parent-uri"]) && ($related != '')) {
self::FetchRelated($related, $item["parent-uri"], $importer); self::FetchRelated($related, $item["parent-uri"], $importer);
$item["type"] = 'remote-comment'; $item["type"] = 'remote-comment';
@ -593,13 +595,143 @@ class ostatus {
$item["parent-uri"] = $item["uri"]; $item["parent-uri"] = $item["uri"];
} }
if ($item['author-link'] != '') { if (($item['author-link'] != '') && !empty($header["protocol"])) {
$item = store_conversation($item); $item = store_conversation($item);
} }
self::$itemlist[] = $item; self::$itemlist[] = $item;
} }
/**
* @brief Fetch the conversation for posts
*
* @param string $conversation The link to the conversation
* @param string $conversation_uri The conversation in "uri" format
*/
private static function fetchConversation($conversation, $conversation_uri) {
// Ensure that we only store a conversation once in a process
if (isset(self::$conv_list[$conversation])) {
return;
}
self::$conv_list[$conversation] = true;
$conversation_data = z_fetch_url($conversation);
if (!$conversation_data['success']) {
return;
}
$xml = '';
if (stristr($conversation_data['header'], 'Content-Type: application/atom+xml')) {
$xml = $conversation_data['body'];
}
if ($xml == '') {
$doc = new DOMDocument();
if (!@$doc->loadHTML($conversation_data['body'])) {
return;
}
$xpath = new DomXPath($doc);
$links = $xpath->query('//link');
if ($links) {
foreach ($links AS $link) {
$attribute = ostatus::read_attributes($link);
if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) {
$file = $attribute['href'];
}
}
if ($file != '') {
$conversation_atom = z_fetch_url($attribute['href']);
if ($conversation_atom['success']) {
$xml = $conversation_atom['body'];
}
}
}
}
if ($xml == '') {
return;
}
self::storeConversation($xml, $conversation, $conversation_uri);
}
/**
* @brief Store a feed in several conversation entries
*
* @param string $xml The feed
*/
private static function storeConversation($xml, $conversation = '', $conversation_uri = '') {
$doc = new DOMDocument();
@$doc->loadXML($xml);
$xpath = new DomXPath($doc);
$xpath->registerNamespace('atom', NAMESPACE_ATOM1);
$xpath->registerNamespace('thr', NAMESPACE_THREAD);
$xpath->registerNamespace('ostatus', NAMESPACE_OSTATUS);
$entries = $xpath->query('/atom:feed/atom:entry');
// Now store the entries
foreach ($entries AS $entry) {
$doc2 = new DOMDocument();
$doc2->preserveWhiteSpace = false;
$doc2->formatOutput = true;
$conv_data = array();
$conv_data['protocol'] = PROTOCOL_SPLITTED_CONV;
$conv_data['network'] = NETWORK_OSTATUS;
$conv_data['uri'] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue;
$inreplyto = $xpath->query('thr:in-reply-to', $entry);
if (is_object($inreplyto->item(0))) {
foreach ($inreplyto->item(0)->attributes AS $attributes) {
if ($attributes->name == "ref") {
$conv_data['reply-to-uri'] = $attributes->textContent;
}
}
}
$conv = $xpath->query('ostatus:conversation/text()', $entry)->item(0)->nodeValue;
$conv_data['conversation-uri'] = $conv;
$conv = $xpath->query('ostatus:conversation', $entry);
if (is_object($conv->item(0))) {
foreach ($conv->item(0)->attributes AS $attributes) {
if ($attributes->name == "ref") {
$conv_data['conversation-uri'] = $attributes->textContent;
}
if ($attributes->name == "href") {
$conv_data['conversation-href'] = $attributes->textContent;
}
}
}
if ($conversation != '') {
$conv_data['conversation-uri'] = $conversation;
}
if ($conversation_uri != '') {
$conv_data['conversation-uri'] = $conversation_uri;
}
$entry = $doc2->importNode($entry, true);
$doc2->appendChild($entry);
$conv_data['source'] = $doc2->saveXML();
logger('Store conversation data for uri '.$conv_data['uri'], LOGGER_DEBUG);
store_conversation($conv_data);
}
}
/** /**
* @brief Fetch related posts and processes them * @brief Fetch related posts and processes them
* *
@ -686,6 +818,17 @@ class ostatus {
} }
} }
// Finally we take the data that we fetched from "ostatus:conversation"
if ($xml == '') {
$condition = array('item-uri' => $related_uri, 'protocol' => PROTOCOL_SPLITTED_CONV);
$conversation = dba::select('conversation', array('source'), $condition, array('limit' => 1));
if (dbm::is_result($conversation)) {
$stored = true;
logger('Got cached XML from conversation for URI '.$related_uri, LOGGER_DEBUG);
$xml = $conversation['source'];
}
}
if ($xml != '') { if ($xml != '') {
self::process($xml, $importer, $contact, $hub, $stored, false); self::process($xml, $importer, $contact, $hub, $stored, false);
} else { } else {