From a3c497190d80795e8f544079f8da7b5563f1ab87 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 7 Jun 2015 23:18:02 +0200 Subject: [PATCH] Complete new OStatus import. --- include/items.php | 68 +++--- include/ostatus.php | 358 +++++++++++++++++++++++++++++++ include/ostatus_conversation.php | 29 +-- mod/item.php | 24 ++- 4 files changed, 438 insertions(+), 41 deletions(-) create mode 100644 include/ostatus.php diff --git a/include/items.php b/include/items.php index c34859d940..97eb15efcf 100644 --- a/include/items.php +++ b/include/items.php @@ -9,10 +9,11 @@ require_once('include/tags.php'); require_once('include/files.php'); require_once('include/text.php'); require_once('include/email.php'); -require_once('include/ostatus_conversation.php'); +//require_once('include/ostatus_conversation.php'); require_once('include/threads.php'); require_once('include/socgraph.php'); require_once('include/plaintext.php'); +require_once('include/ostatus.php'); require_once('mod/share.php'); function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0, $forpubsub = false) { @@ -119,7 +120,7 @@ function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0, // Include answers to status.net posts in pubsub feeds if($forpubsub) { $sql_post_table = "INNER JOIN `thread` ON `thread`.`iid` = `item`.`parent` "; - $visibility = sprintf("OR (`item`.`network` = '%s' AND `thread`.`network`='%s')", + $visibility = sprintf("AND (`item`.`parent` = `item`.`id`) OR (`item`.`network` = '%s' AND `thread`.`network`='%s')", dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS)); $date_field = "`received`"; $sql_order = "`item`.`received` DESC"; @@ -885,22 +886,22 @@ function get_atom_elements($feed, $item, $contact = array()) { } } - // Search for ostatus conversation url - $links = $item->feed->data["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["feed"][0]["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["entry"][0]["child"]["http://www.w3.org/2005/Atom"]["link"]; - - if (is_array($links)) { - foreach ($links as $link) { - $conversation = array_shift($link["attribs"]); - - if ($conversation["rel"] == "ostatus:conversation") { - $res["ostatus_conversation"] = ostatus_convert_href($conversation["href"]); - logger('get_atom_elements: found conversation url '.$res["ostatus_conversation"]); - //} elseif ($conversation["rel"] == "alternate") { - // $res["plink"] = $conversation["href"]; - // logger('get_atom_elements: found plink '.$res["plink"]); - } - }; - } +// // Search for ostatus conversation url +// $links = $item->feed->data["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["feed"][0]["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["entry"][0]["child"]["http://www.w3.org/2005/Atom"]["link"]; +// +// if (is_array($links)) { +// foreach ($links as $link) { +// $conversation = array_shift($link["attribs"]); +// +// if ($conversation["rel"] == "ostatus:conversation") { +// $res["ostatus_conversation"] = ostatus_convert_href($conversation["href"]); +// logger('get_atom_elements: found conversation url '.$res["ostatus_conversation"]); +// //} elseif ($conversation["rel"] == "alternate") { +// // $res["plink"] = $conversation["href"]; +// // logger('get_atom_elements: found plink '.$res["plink"]); +// } +// }; +// } if (isset($contact["network"]) AND ($contact["network"] == NETWORK_FEED) AND $contact['fetch_further_information']) { $preview = ""; @@ -1138,14 +1139,14 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa $arr['plink'] = ostatus_convert_href($arr['uri']); } - // if an OStatus conversation url was passed in, it is stored and then - // removed from the array. - $ostatus_conversation = null; +// // if an OStatus conversation url was passed in, it is stored and then +// // removed from the array. +// $ostatus_conversation = null; - if (isset($arr["ostatus_conversation"])) { - $ostatus_conversation = $arr["ostatus_conversation"]; - unset($arr["ostatus_conversation"]); - } +// if (isset($arr["ostatus_conversation"])) { +// $ostatus_conversation = $arr["ostatus_conversation"]; +// unset($arr["ostatus_conversation"]); +// } if(x($arr, 'gravity')) $arr['gravity'] = intval($arr['gravity']); @@ -1386,7 +1387,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa $arr['gravity'] = 0; } else { - logger('item_store: item parent was not found - ignoring item'); + logger('item_store: item parent '.$arr['parent-uri'].' for '.$arr['uid'].' was not found - ignoring item'); return 0; } @@ -1525,8 +1526,8 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa ); // Complete ostatus threads - if ($ostatus_conversation) - complete_conversation($current_post, $ostatus_conversation); + //if ($ostatus_conversation) + // complete_conversation($current_post, $ostatus_conversation); $arr['id'] = $current_post; $arr['parent'] = $parent_id; @@ -2224,6 +2225,13 @@ function edited_timestamp_is_newer($existing, $update) { */ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0) { + if ($contact['network'] === NETWORK_OSTATUS) { + if ($pass < 2) { + logger("Consume OStatus messages ", LOGGER_DEBUG); + ostatus_import($xml,$importer,$contact); + } + return; + } require_once('library/simplepie/simplepie.inc'); require_once('include/contact_selectors.php'); @@ -4403,6 +4411,10 @@ function atom_entry($item,$type,$author,$owner,$comment = false,$cid = 0) { } } + //$o .= ''."\r\n"; + //$o .= ''."\r\n"; + //$o .= ''."\r\n"; + $o .= item_get_attachment($item); $o .= item_getfeedattach($item); diff --git a/include/ostatus.php b/include/ostatus.php new file mode 100644 index 0000000000..c426a9f10e --- /dev/null +++ b/include/ostatus.php @@ -0,0 +1,358 @@ +evaluate('atom:author/atom:uri/text()', $context)->item(0)->nodeValue; + $author["author-name"] = $xpath->evaluate('atom:author/atom:name/text()', $context)->item(0)->nodeValue; + + // Preserve the value + $authorlink = $author["author-link"]; + + $alternate = $xpath->query("atom:author/atom:link[@rel='alternate']", $context)->item(0)->attributes; + if (is_object($alternate)) + foreach($alternate AS $attributes) + if ($attributes->name == "href") + $author["author-link"] = $attributes->textContent; + + $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `nurl` IN ('%s', '%s') AND `network` != '%s'", + intval($importer["uid"]), dbesc(normalise_link($author["author-link"])), + dbesc(normalise_link($authorlink)), dbesc(NETWORK_STATUSNET)); + if ($r) { + $contact = $r[0]; + $author["contact-id"] = $r[0]["id"]; + } else + $author["contact-id"] = $contact["id"]; + + $avatarlist = array(); + $avatars = $xpath->query("atom:author/atom:link[@rel='avatar']", $context); + foreach($avatars AS $avatar) { + $href = ""; + $width = 0; + foreach($avatar->attributes AS $attributes) { + if ($attributes->name == "href") + $href = $attributes->textContent; + if ($attributes->name == "width") + $width = $attributes->textContent; + } + if (($width > 0) AND ($href != "")) + $avatarlist[$width] = $href; + } + if (count($avatarlist) > 0) { + krsort($avatarlist); + $author["author-avatar"] = current($avatarlist); + } + + $displayname = $xpath->evaluate('atom:author/poco:displayName/text()', $context)->item(0)->nodeValue; + if ($displayname != "") + $author["author-name"] = $displayname; + + $author["owner-name"] = $author["author-name"]; + $author["owner-link"] = $author["author-link"]; + $author["owner-avatar"] = $author["author-avatar"]; + + return($author); +} + +function ostatus_import($xml,$importer,&$contact) { + + $a = get_app(); + + logger("Import OStatus message", LOGGER_DEBUG); + + if ($xml == "") + return; + + $doc = new DOMDocument(); + $doc->loadXML($xml); + + $xpath = new DomXPath($doc); + $xpath->registerNamespace('atom', "http://www.w3.org/2005/Atom"); + $xpath->registerNamespace('thr', "http://purl.org/syndication/thread/1.0"); + $xpath->registerNamespace('georss', "http://www.georss.org/georss"); + $xpath->registerNamespace('activity', "http://activitystrea.ms/spec/1.0/"); + $xpath->registerNamespace('media', "http://purl.org/syndication/atommedia"); + $xpath->registerNamespace('poco', "http://portablecontacts.net/spec/1.0"); + $xpath->registerNamespace('ostatus', "http://ostatus.org/schema/1.0"); + $xpath->registerNamespace('statusnet', "http://status.net/schema/api/1/"); + + $header = array(); + $header["uid"] = $importer["uid"]; + $header["network"] = NETWORK_OSTATUS; + $header["type"] = "remote"; + $header["wall"] = 0; + $header["origin"] = 0; + $header["gravity"] = GRAVITY_PARENT; + + // it could either be a received post or a post we fetched by ourselves + // depending on that, the first node is different + $first_child = $doc->firstChild->tagName; + + if ($first_child == "feed") + $entries = $xpath->query('/atom:feed/atom:entry'); + else + $entries = $xpath->query('/atom:entry'); + + $conversation = ""; + $conversationlist = array(); + $item_id = 0; + + // Reverse the order of the entries + foreach ($entries AS $entry) + $entrylist[] = $entry; + + foreach (array_reverse($entrylist) AS $entry) { + + $mention = false; + + // fetch the author + if ($first_child == "feed") + $author = ostatus_fetchauthor($xpath, $doc->firstChild, $contact); + else + $author = ostatus_fetchauthor($xpath, $entry, $contact); + + $item = array_merge($header, $author); + + // Now get the item + $item["uri"] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue; + $item["body"] = html2bbcode($xpath->query('atom:content/text()', $entry)->item(0)->nodeValue); + $item["object-type"] = $xpath->query('activity:object-type/text()', $entry)->item(0)->nodeValue; + $item["verb"] = $xpath->query('activity:verb/text()', $entry)->item(0)->nodeValue; + + if ($item["verb"] == ACTIVITY_FOLLOW) { + // ignore "Follow" messages + $item = array(); + continue; + } + + if ($item["verb"] == ACTIVITY_FAVORITE) { + // ignore "Favorite" messages + $item = array(); + continue; + } + + $item["created"] = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue; + $item["edited"] = $xpath->query('atom:updated/text()', $entry)->item(0)->nodeValue; + $conversation = $xpath->query('ostatus:conversation/text()', $entry)->item(0)->nodeValue; + + $related = ""; + + $inreplyto = $xpath->query('thr:in-reply-to', $entry); + if (is_object($inreplyto->item(0))) { + foreach($inreplyto->item(0)->attributes AS $attributes) { + if ($attributes->name == "ref") + $item["parent-uri"] = $attributes->textContent; + if ($attributes->name == "href") + $related = $attributes->textContent; + } + } + + $georsspoint = $xpath->query('georss:point', $entry); + if ($georsspoint) + $item["coord"] = $georsspoint->item(0)->nodeValue; + + $categories = $xpath->query('atom:category', $entry); + if ($categories) { + foreach ($categories AS $category) { + foreach($category->attributes AS $attributes) + if ($attributes->name == "term") { + $term = $attributes->textContent; + if(strlen($item["tag"])) + $item["tag"] .= ','; + $item["tag"] .= "#[url=".$a->get_baseurl()."/search?tag=".$term."]".$term."[/url]"; + } + } + } + + $self = ""; + $enclosure = ""; + + $links = $xpath->query('atom:link', $entry); + if ($links) { + $rel = ""; + $href = ""; + $type = ""; + $length = "0"; + $title = ""; + foreach ($links AS $link) { + foreach($link->attributes AS $attributes) { + if ($attributes->name == "href") + $href = $attributes->textContent; + if ($attributes->name == "rel") + $rel = $attributes->textContent; + if ($attributes->name == "type") + $type = $attributes->textContent; + if ($attributes->name == "length") + $length = $attributes->textContent; + if ($attributes->name == "title") + $title = $attributes->textContent; + } + if (($rel != "") AND ($href != "")) + switch($rel) { + case "alternate": + $item["plink"] = $href; + break; + case "ostatus:conversation": + $conversation = $href; + break; + case "enclosure": + $enclosure = $href; + if(strlen($item["attach"])) + $item["attach"] .= ','; + + $item["attach"] .= '[attach]href="'.$href.'" length="'.$length.'" type="'.$type.'" title="'.$title.'"[/attach]'; + break; + case "related": + if (!isset($item["parent-uri"])) + $item["parent-uri"] = $href; + + if ($related == "") + $related = $href; + break; + case "self": + $self = $href; + break; + case "mentioned": + // Notification check + if ($importer["nurl"] == normalise_link($href)) + $mention = true; + break; + } + } + } + + $local_id = ""; + $repeat_of = ""; + + $notice_info = $xpath->query('statusnet:notice_info', $entry); + if ($notice_info) + foreach($notice_info->item(0)->attributes AS $attributes) { + if ($attributes->name == "source") + $item["app"] = strip_tags($attributes->textContent); + if ($attributes->name == "local_id") + $local_id = $attributes->textContent; + if ($attributes->name == "repeat_of") + $repeat_of = $attributes->textContent; + } + + // Is it a repeated post? + if ($repeat_of != "") { + $activityobjects = $xpath->query('activity:object', $entry)->item(0); + + if (is_object($activityobjects)) { + + $orig_uris = $xpath->query("activity:object/atom:link[@rel='alternate']", $activityobjects); + if ($orig_uris) + foreach($orig_uris->item(0)->attributes AS $attributes) + if ($attributes->name == "href") + $orig_uri = $attributes->textContent; + + if (!isset($orig_uri)) + $orig_uri = $xpath->query("atom:link[@rel='alternate']", $activityobjects)->item(0)->nodeValue; + + if (!isset($orig_uri)) + $orig_uri = $xpath->query("activity:object/atom:id", $activityobjects)->item(0)->nodeValue; + + if (!isset($orig_uri)) + $orig_uri = $xpath->query('atom:id/text()', $activityobjects)->item(0)->nodeValue; + + $orig_body = $xpath->query('atom:content/text()', $activityobjects)->item(0)->nodeValue; + $orig_created = $xpath->query('atom:published/text()', $activityobjects)->item(0)->nodeValue; + + $orig_contact = $contact; + $orig_author = ostatus_fetchauthor($xpath, $activityobjects, $orig_contact); + + $prefix = share_header($orig_author['author-name'], $orig_author['author-link'], $orig_author['author-avatar'], "", $orig_created, $orig_uri); + $item["body"] = $prefix.html2bbcode($orig_body)."[/share]"; + + $item["verb"] = $xpath->query('activity:verb/text()', $activityobjects)->item(0)->nodeValue; + $item["object-type"] = $xpath->query('activity:object-type/text()', $activityobjects)->item(0)->nodeValue; + } + } + + if ($enclosure != "") + $item["body"] .= add_page_info($enclosure); + + if (isset($item["parent-uri"])) { + $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s'", + intval($importer["uid"]), dbesc($item["parent-uri"])); + + if (!$r AND ($related != "")) { + $reply_path = str_replace("/notice/", "/api/statuses/show/", $related).".atom"; + + if ($reply_path != $related) { + logger("Fetching related items for user ".$importer["uid"]." from ".$reply_path, LOGGER_DEBUG); + $reply_xml = fetch_url($reply_path); + + $reply_contact = $contact; + ostatus_import($reply_xml,$importer,$reply_contact); + + // After the import try to fetch the parent item again + $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s'", + intval($importer["uid"]), dbesc($item["parent-uri"])); + } + } + if ($r) { + $item["type"] = 'remote-comment'; + $item["gravity"] = GRAVITY_COMMENT; + } + } else + $item["parent-uri"] = $item["uri"]; + + $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s'", + intval($importer["uid"]), dbesc($item["uri"])); + if (!$r) { + $item_id = item_store($item); + //echo $xml; + //print_r($item); + //echo $item_id." ".$item["parent-uri"]."\n"; + + if ($item_id) + logger("Item was stored with id ".$item_id, LOGGER_DEBUG); + else + logger("Error storing item ".print_r($item, true), LOGGER_DEBUG); + + $item["id"] = $item_id; + + if (!isset($item["parent"]) OR ($item["parent"] == 0)) + $item["parent"] = $item_id; + + if ($mention AND ($item["id"] != 0)) { + $u = q("SELECT `notify-flags`, `language`, `username`, `email` FROM user WHERE uid = %d LIMIT 1", intval($item['uid'])); + + notification(array( + 'type' => NOTIFY_TAGSELF, + 'notify_flags' => $u[0]["notify-flags"], + 'language' => $u[0]["language"], + 'to_name' => $u[0]["username"], + 'to_email' => $u[0]["email"], + 'uid' => $item["uid"], + 'item' => $item, + 'link' => $a->get_baseurl().'/display/'.urlencode(get_item_guid($item["id"])), + 'source_name' => $item["author-name"], + 'source_link' => $item["author-link"], + 'source_photo' => $item["author-avatar"], + 'verb' => ACTIVITY_TAG, + 'otype' => 'item', + 'parent' => $item["parent"] + )); + } + } else { + $item_id = $r[0]["id"]; + logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$r[0]["id"], LOGGER_DEBUG); + } + + if (($conversation != "") AND ($item_id != 0)) { + // Check for duplicates. We really don't need to check the same conversation twice. + if (!in_array($conversation, $conversationlist)) { + complete_conversation($item_id, $conversation); + $conversationlist[] = $conversation; + } + } + } +} diff --git a/include/ostatus_conversation.php b/include/ostatus_conversation.php index 2670d19766..9642ae1dbf 100644 --- a/include/ostatus_conversation.php +++ b/include/ostatus_conversation.php @@ -138,13 +138,13 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio if (isset($single_conv->object->id)) $single_conv->id = $single_conv->object->id; - logger("Got id ".$single_conv->id, LOGGER_DEBUG); + //logger("Got id ".$single_conv->id, LOGGER_DEBUG); $plink = ostatus_convert_href($single_conv->id); if (isset($single_conv->object->url)) $plink = ostatus_convert_href($single_conv->object->url); - logger("Got url ".$plink, LOGGER_DEBUG); + //logger("Got url ".$plink, LOGGER_DEBUG); if (@!$single_conv->id) continue; @@ -157,7 +157,8 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); if ($new_parents) { $parent = $new_parents[0]; - logger('adopting new parent '.$parent["id"].' for '.$itemid); + if ($parent["id"] != $message["parent"]) + logger('Fetch new parent id '.$parent["id"].' for '.$itemid.' Old parent: '.$message["parent"]); } else { $parent["id"] = 0; $parent["uri"] = $first_id; @@ -169,12 +170,12 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio else $parent_uri = $parent["uri"]; - $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `plink` = '%s' AND `network` IN ('%s','%s') LIMIT 1", + $message_exists = q("SELECT `id`, `parent` FROM `item` WHERE `uid` = %d AND `plink` = '%s' AND `network` IN ('%s','%s') LIMIT 1", intval($message["uid"]), dbesc($plink), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); if (!$message_exists) - $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", + $message_exists = q("SELECT `id`, `parent` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", intval($message["uid"]), dbesc($single_conv->id), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); @@ -182,14 +183,17 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio if ($parent["id"] != 0) { $existing_message = $message_exists[0]; - logger('updating id '.$existing_message["id"].' to parent '.$parent["id"].' uri '.$parent["uri"].' thread '.$parent_uri, LOGGER_DEBUG); + // Normally this shouldn't happen anymore, since we improved the way we fetch OStatus messages + if ($existing_message["parent"] != $parent["id"]) { + logger('updating id '.$existing_message["id"].' to parent '.$parent["id"].' uri '.$parent["uri"].' thread '.$parent_uri, LOGGER_DEBUG); - // This is partly bad, since the entry in the thread table isn't updated - $r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `thr-parent` = '%s' WHERE `id` = %d", - intval($parent["id"]), - dbesc($parent["uri"]), - dbesc($parent_uri), - intval($existing_message["id"])); + // This is partly bad, since the entry in the thread table isn't updated + $r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `thr-parent` = '%s' WHERE `id` = %d", + intval($parent["id"]), + dbesc($parent["uri"]), + dbesc($parent_uri), + intval($existing_message["id"])); + } } continue; } @@ -274,6 +278,7 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio complete_conversation($newitem, $conversation_url, true); // If the newly created item is the top item then change the parent settings of the thread + // This shouldn't happen anymore. This could is supposed to be absolote. if ($newitem AND ($arr["uri"] == $first_id)) { logger('setting new parent to id '.$newitem); $new_parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `id` = %d LIMIT 1", diff --git a/mod/item.php b/mod/item.php index 184920d4ce..7775d2950e 100644 --- a/mod/item.php +++ b/mod/item.php @@ -1097,8 +1097,30 @@ function handle_tag($a, &$body, &$inform, &$str_tags, $profile_uid, $tag, $netwo //is it a person tag? if(strpos($tag,'@') === 0) { //is it already replaced? - if(strpos($tag,'[url=')) + if(strpos($tag,'[url=')) { + //append tag to str_tags + if(!stristr($str_tags,$tag)) { + if(strlen($str_tags)) + $str_tags .= ','; + $str_tags .= $tag; + } + + // Checking for the alias that is used for OStatus + $pattern = "/@\[url\=(.*?)\](.*?)\[\/url\]/ism"; + if (preg_match($pattern, $tag, $matches)) { + $data = probe_url($matches[1]); + if ($data["alias"] != "") { + $newtag = '@[url='.$data["alias"].']'.$data["name"].'[/url]'; + if(!stristr($str_tags,$newtag)) { + if(strlen($str_tags)) + $str_tags .= ','; + $str_tags .= $newtag; + } + } + } + return $replaced; + } $stat = false; //get the person's name $name = substr($tag,1);