From 0c752a2190e5adecf4b83e6a204d3f0dddf32150 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 25 May 2015 13:27:45 +0200 Subject: [PATCH 1/6] Diaspora: Store the original XML and the object type. --- include/diaspora.php | 58 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/include/diaspora.php b/include/diaspora.php index 04588dffe6..c683ded1da 100755 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -833,12 +833,19 @@ function diaspora_post($importer,$xml,$msg) { $body = diaspora2bb($xml->raw_message); - // Add OEmbed and other information to the body - if (!diaspora_is_redmatrix($contact['url'])) - $body = add_page_info_to_body($body, false, true); - $datarray = array(); + $datarray["object"] = json_encode($xml); + + if($xml->photo->remote_photo_path AND $xml->photo->remote_photo_name) + $datarray["object-type"] = ACTIVITY_OBJ_PHOTO; + else { + $datarray['object-type'] = ACTIVITY_OBJ_NOTE; + // Add OEmbed and other information to the body + if (!diaspora_is_redmatrix($contact['url'])) + $body = add_page_info_to_body($body, false, true); + } + $str_tags = ''; $cnt = preg_match_all('/@\[url=(.*?)\[\/url\]/ism',$body,$matches,PREG_SET_ORDER); @@ -872,7 +879,10 @@ function diaspora_post($importer,$xml,$msg) { $datarray['author-avatar'] = $contact['thumb']; $datarray['body'] = $body; $datarray['tag'] = $str_tags; - $datarray['app'] = 'Diaspora'; + if ($xml->provider_display_name) + $datarray["app"] = unxmlify($xml->provider_display_name); + else + $datarray['app'] = 'Diaspora'; // if empty content it might be a photo that hasn't arrived yet. If a photo arrives, we'll make it visible. @@ -921,6 +931,8 @@ function diaspora_store_by_guid($guid, $server, $uid = 0) { $author = $item["author"]; $guid = $item["guid"]; $private = $item["private"]; + $object = $item["object"]; + $objecttype = $item["object-type"]; $message_id = $author.':'.$guid; $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `guid` = '%s' LIMIT 1", @@ -954,6 +966,8 @@ function diaspora_store_by_guid($guid, $server, $uid = 0) { $datarray['tag'] = $str_tags; $datarray['app'] = $app; $datarray['visible'] = ((strlen($body)) ? 1 : 0); + $datarray['object'] = $object; + $datarray['object-type'] = $objecttype; if ($datarray['contact-id'] == 0) return false; @@ -1001,11 +1015,14 @@ function diaspora_fetch_message($guid, $server, $level = 0) { $item["guid"] = unxmlify($source_xml->post->status_message->guid); $item["private"] = (unxmlify($source_xml->post->status_message->public) == 'false'); + $item["object"] = json_encode($source_xml->post); if(strlen($source_xml->post->asphoto->objectId) && ($source_xml->post->asphoto->objectId != 0) && ($source_xml->post->asphoto->image_url)) { + $item["object-type"] = ACTIVITY_OBJ_PHOTO; $body = '[url=' . notags(unxmlify($source_xml->post->asphoto->image_url)) . '][img]' . notags(unxmlify($source_xml->post->asphoto->objectId)) . '[/img][/url]' . "\n"; $body = scale_external_images($body,false); } elseif($source_xml->post->asphoto->image_url) { + $item["object-type"] = ACTIVITY_OBJ_PHOTO; $body = '[img]' . notags(unxmlify($source_xml->post->asphoto->image_url)) . '[/img]' . "\n"; $body = scale_external_images($body); } elseif($source_xml->post->status_message) { @@ -1015,18 +1032,25 @@ function diaspora_fetch_message($guid, $server, $level = 0) { if($source_xml->post->status_message->photo->remote_photo_path AND $source_xml->post->status_message->photo->remote_photo_name) { + $item["object-type"] = ACTIVITY_OBJ_PHOTO; + $remote_photo_path = notags(unxmlify($source_xml->post->status_message->photo->remote_photo_path)); $remote_photo_name = notags(unxmlify($source_xml->post->status_message->photo->remote_photo_name)); $body = '[img]'.$remote_photo_path.$remote_photo_name.'[/img]'."\n".$body; logger('embedded picture link found: '.$body, LOGGER_DEBUG); - } + } else + $item["object-type"] = ACTIVITY_OBJ_NOTE; $body = scale_external_images($body); // Add OEmbed and other information to the body - $body = add_page_info_to_body($body, false, true); + // To-Do: It could be a repeated redmatrix item + // Then we shouldn't add further data to it + if ($item["object-type"] == ACTIVITY_OBJ_NOTE) + $body = add_page_info_to_body($body, false, true); + } elseif($source_xml->post->reshare) { // Reshare of a reshare return diaspora_fetch_message($source_xml->post->reshare->root_guid, $server, ++$level); @@ -1105,6 +1129,8 @@ function diaspora_reshare($importer,$xml,$msg) { $orig_author = $r[0]["author-link"]; $create_original_post = ($body != ""); $orig_url = $a->get_baseurl()."/display/".$orig_guid; + $object = $r[0]["object"]; + $objecttype = $r[0]["object-type"]; } } @@ -1144,6 +1170,8 @@ function diaspora_reshare($importer,$xml,$msg) { $orig_guid = $item["guid"]; $create_original_post = ($body != ""); $orig_url = $a->get_baseurl()."/display/".$orig_guid; + $object = $item["object"]; + $objecttype = $item["object-type"]; } } @@ -1184,6 +1212,9 @@ function diaspora_reshare($importer,$xml,$msg) { $datarray['body'] = $body; } + $datarray["object"] = json_encode($xml); + $datarray['object-type'] = $objecttype; + $datarray['tag'] = $str_tags; $datarray['app'] = $app; @@ -1210,6 +1241,7 @@ function diaspora_reshare($importer,$xml,$msg) { $datarray2['owner-link'] = $datarray2['author-link']; $datarray2['owner-avatar'] = $datarray2['author-avatar']; $datarray2['body'] = $body; + $datarray2["object"] = $object; DiasporaFetchGuid($datarray2); $message_id = item_store($datarray2); @@ -1295,6 +1327,8 @@ function diaspora_asphoto($importer,$xml,$msg) { $datarray['author-link'] = $contact['url']; $datarray['author-avatar'] = $contact['thumb']; $datarray['body'] = $body; + $datarray["object"] = json_encode($xml); + $datarray['object-type'] = ACTIVITY_OBJ_PHOTO; $datarray['app'] = 'Diaspora/Cubbi.es'; @@ -1464,6 +1498,8 @@ function diaspora_comment($importer,$xml,$msg) { $datarray['author-link'] = $person['url']; $datarray['author-avatar'] = ((x($person,'thumb')) ? $person['thumb'] : $person['photo']); $datarray['body'] = $body; + $datarray["object"] = json_encode($xml); + $datarray["object-type"] = ACTIVITY_OBJ_COMMENT; // We can't be certain what the original app is if the message is relayed. if(($parent_item['origin']) && (! $parent_author_signature)) @@ -1790,7 +1826,7 @@ function diaspora_message($importer,$xml,$msg) { dbesc($person['name']), dbesc($person['photo']), dbesc($person['url']), - intval($contact['id']), + intval($contact['id']), dbesc($conversation['subject']), dbesc($body), 0, @@ -2059,7 +2095,7 @@ function diaspora_like($importer,$xml,$msg) { $activity = ACTIVITY_LIKE; $post_type = (($parent_item['resource-id']) ? t('photo') : t('status')); - $objtype = (($parent_item['resource-id']) ? ACTIVITY_OBJ_PHOTO : ACTIVITY_OBJ_NOTE ); + $objtype = (($parent_item['resource-id']) ? ACTIVITY_OBJ_PHOTO : ACTIVITY_OBJ_NOTE ); $link = xmlify('' . "\n") ; $body = $parent_item['body']; @@ -2878,7 +2914,7 @@ function diaspora_send_retraction($item,$owner,$contact,$public_batch = false) { $target_type = (($item['verb'] === ACTIVITY_LIKE) ? 'Like' : 'Comment'); } else { - + $tpl = get_markup_template('diaspora_signed_retract.tpl'); $target_type = 'StatusMessage'; } @@ -2924,7 +2960,7 @@ function diaspora_send_mail($item,$owner,$contact) { $body = bb2diaspora($item['body']); $created = datetime_convert('UTC','UTC',$item['created'],'Y-m-d H:i:s \U\T\C'); - + $signed_text = $item['guid'] . ';' . $cnv['guid'] . ';' . $body . ';' . $created . ';' . $myaddr . ';' . $cnv['guid']; From 7e401ca199ce9c47c8b5c8ae96a476023be9a048 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 25 May 2015 19:07:59 +0200 Subject: [PATCH 2/6] Improved check for duplicates. Improvements with reshared Diaspora items. --- include/diaspora.php | 33 +++++++++++++++++++++------------ include/items.php | 15 ++++++++++++++- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/include/diaspora.php b/include/diaspora.php index c683ded1da..68e38f8c4c 100755 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -785,12 +785,21 @@ function diaspora_is_redmatrix($url) { } function diaspora_plink($addr, $guid) { - $r = q("SELECT `url`, `nick` FROM `fcontact` WHERE `addr`='%s' LIMIT 1", $addr); + $r = q("SELECT `url`, `nick`, `network` FROM `fcontact` WHERE `addr`='%s' LIMIT 1", $addr); // Fallback if (!$r) return 'https://'.substr($addr,strpos($addr,'@')+1).'/posts/'.$guid; + // Friendica contacts are often detected as Diaspora contacts in the "fcontact" table + // So we try another way as well. + $s = q("SELECT `network` FROM `gcontact` WHERE `nurl`='%s' LIMIT 1", dbesc(normalise_link($r[0]["url"]))); + if ($s) + $r[0]["network"] = $s[0]["network"]; + + if ($r[0]["network"] == NETWORK_DFRN) + return(str_replace("/profile/".$r[0]["nick"]."/", "/display/".$guid, $r[0]["url"]."/")); + if (diaspora_is_redmatrix($r[0]["url"])) return $r[0]["url"]."/?f=&mid=".$guid; @@ -1105,16 +1114,17 @@ function diaspora_reshare($importer,$xml,$msg) { $orig_author = notags(unxmlify($xml->root_diaspora_id)); $orig_guid = notags(unxmlify($xml->root_guid)); + $orig_url = $a->get_baseurl()."/display/".$orig_guid; $create_original_post = false; // Do we already have this item? - $r = q("SELECT `body`, `tag`, `app`, `created`, `author-link`, `plink` FROM `item` WHERE `guid` = '%s' AND `visible` AND NOT `deleted` AND `body` != '' LIMIT 1", + $r = q("SELECT `body`, `tag`, `app`, `created`, `plink`, `object`, `object-type`, `uri` FROM `item` WHERE `guid` = '%s' AND `visible` AND NOT `deleted` AND `body` != '' LIMIT 1", dbesc($orig_guid), dbesc(NETWORK_DIASPORA) ); if(count($r)) { - logger('reshared message '.$orig_guid." reshared by ".$guid.' already exists on system: '.$orig_url); + logger('reshared message '.$orig_guid." reshared by ".$guid.' already exists on system.'); // Maybe it is already a reshared item? // Then refetch the content, since there can be many side effects with reshared posts from other networks or reshares from reshares @@ -1126,9 +1136,9 @@ function diaspora_reshare($importer,$xml,$msg) { $str_tags = $r[0]["tag"]; $app = $r[0]["app"]; $orig_created = $r[0]["created"]; - $orig_author = $r[0]["author-link"]; + $orig_plink = $r[0]["plink"]; + $orig_uri = $r[0]["uri"]; $create_original_post = ($body != ""); - $orig_url = $a->get_baseurl()."/display/".$orig_guid; $object = $r[0]["object"]; $objecttype = $r[0]["object-type"]; } @@ -1139,8 +1149,6 @@ function diaspora_reshare($importer,$xml,$msg) { $str_tags = ""; $app = ""; - $orig_url = 'https://'.substr($orig_author,strpos($orig_author,'@')+1).'/posts/'.$orig_guid; - $server = 'https://'.substr($orig_author,strpos($orig_author,'@')+1); logger('1st try: reshared message '.$orig_guid." reshared by ".$guid.' will be fetched from original server: '.$server); $item = diaspora_fetch_message($orig_guid, $server); @@ -1168,13 +1176,16 @@ function diaspora_reshare($importer,$xml,$msg) { $orig_created = $item["created"]; $orig_author = $item["author"]; $orig_guid = $item["guid"]; + $orig_plink = diaspora_plink($orig_author, $orig_guid); + $orig_uri = $orig_author.':'.$orig_guid; $create_original_post = ($body != ""); - $orig_url = $a->get_baseurl()."/display/".$orig_guid; $object = $item["object"]; $objecttype = $item["object-type"]; } } + $plink = diaspora_plink($diaspora_handle, $guid); + $person = find_diaspora_person_by_handle($orig_author); $created = unxmlify($xml->created_at); @@ -1182,8 +1193,6 @@ function diaspora_reshare($importer,$xml,$msg) { $datarray = array(); - $plink = diaspora_plink($diaspora_handle, $guid); - $datarray['uid'] = $importer['uid']; $datarray['contact-id'] = $contact['id']; $datarray['wall'] = 0; @@ -1230,9 +1239,9 @@ function diaspora_reshare($importer,$xml,$msg) { $datarray2['uid'] = 0; $datarray2['contact-id'] = get_contact($person['url'], 0); $datarray2['guid'] = $orig_guid; - $datarray2['uri'] = $datarray2['parent-uri'] = $orig_author.':'.$orig_guid; + $datarray2['uri'] = $orig_uri; $datarray2['changed'] = $datarray2['created'] = $datarray2['edited'] = $datarray2['commented'] = $datarray2['received'] = datetime_convert('UTC','UTC',$orig_created); - $datarray2['plink'] = diaspora_plink($orig_author, $orig_guid); + $datarray2['plink'] = $orig_plink; $datarray2['author-name'] = $person['name']; $datarray2['author-link'] = $person['url']; diff --git a/include/items.php b/include/items.php index 85d02ab60d..58db71d479 100644 --- a/include/items.php +++ b/include/items.php @@ -1351,9 +1351,22 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa dbesc($arr['uri']), intval($arr['uid']) ); + if($r && count($r)) { - logger('item-store: duplicate item ignored. ' . print_r($arr,true)); + logger('duplicated item with the same uri found. ' . print_r($arr,true)); return 0; + } else { + // Check for an existing post with the same content. There seems to be a problem with OStatus. + $r = q("SELECT `id` FROM `item` WHERE `body` = '%s' AND `created` = '%s' AND `contact-id` = %d AND `uid` = %d LIMIT 1", + dbesc($arr['body']), + dbesc($arr['created']), + intval($arr['contact-id']), + intval($arr['uid']) + ); + if($r && count($r)) { + logger('duplicated item with the same body found. ' . print_r($arr,true)); + return 0; + } } // Is this item available in the global items (with uid=0)? From 03214254ff848521ec00d669006061692b02c01d Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 26 May 2015 23:33:38 +0200 Subject: [PATCH 3/6] OStatus: Improved duplicate check. --- include/diaspora.php | 4 +- include/items.php | 12 ++++- include/ostatus_conversation.php | 75 ++++++++++++++++++++++++++------ 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/include/diaspora.php b/include/diaspora.php index 68e38f8c4c..b70cffdc39 100755 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -1138,7 +1138,6 @@ function diaspora_reshare($importer,$xml,$msg) { $orig_created = $r[0]["created"]; $orig_plink = $r[0]["plink"]; $orig_uri = $r[0]["uri"]; - $create_original_post = ($body != ""); $object = $r[0]["object"]; $objecttype = $r[0]["object-type"]; } @@ -1239,8 +1238,9 @@ function diaspora_reshare($importer,$xml,$msg) { $datarray2['uid'] = 0; $datarray2['contact-id'] = get_contact($person['url'], 0); $datarray2['guid'] = $orig_guid; - $datarray2['uri'] = $orig_uri; + $datarray2['uri'] = $datarray2['parent-uri'] = $orig_uri; $datarray2['changed'] = $datarray2['created'] = $datarray2['edited'] = $datarray2['commented'] = $datarray2['received'] = datetime_convert('UTC','UTC',$orig_created); + $datarray2['parent'] = 0; $datarray2['plink'] = $orig_plink; $datarray2['author-name'] = $person['name']; diff --git a/include/items.php b/include/items.php index 58db71d479..3b4a5a193d 100644 --- a/include/items.php +++ b/include/items.php @@ -864,7 +864,7 @@ function get_atom_elements($feed, $item, $contact = array()) { $conversation = array_shift($link["attribs"]); if ($conversation["rel"] == "ostatus:conversation") { - $res["ostatus_conversation"] = $conversation["href"]; + $res["ostatus_conversation"] = ostatus_convert_href($conversation["href"]); logger('get_atom_elements: found conversation url '.$res["ostatus_conversation"]); } }; @@ -1090,6 +1090,14 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa unset($arr['dsprsig']); } + // Converting the plink + if ($arr['network'] == NETWORK_OSTATUS) { + if (isset($arr['plink'])) + $arr['plink'] = ostatus_convert_href($arr['plink']); + elseif (isset($arr['uri'])) + $arr['plink'] = ostatus_convert_href($arr['uri']); + } + // if an OStatus conversation url was passed in, it is stored and then // removed from the array. $ostatus_conversation = null; @@ -1115,7 +1123,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa /* check for create date and expire time */ $uid = intval($arr['uid']); - $r = q("SELECT expire FROM user WHERE uid = %d", $uid); + $r = q("SELECT expire FROM user WHERE uid = %d", intval($uid)); if(count($r)) { $expire_interval = $r[0]['expire']; if ($expire_interval>0) { diff --git a/include/ostatus_conversation.php b/include/ostatus_conversation.php index 667f7dde42..da8dda292d 100644 --- a/include/ostatus_conversation.php +++ b/include/ostatus_conversation.php @@ -2,7 +2,30 @@ define('OSTATUS_DEFAULT_POLL_INTERVAL', 30); // given in minutes define('OSTATUS_DEFAULT_POLL_TIMEFRAME', 1440); // given in minutes -function check_conversations() { +function ostatus_convert_href($href) { + $elements = explode(":",$href); + + if ((count($elements) <= 2) OR ($elements[0] != "tag")) + return $href; + + $server = explode(",", $elements[1]); + $conversation = explode("=", $elements[2]); + + if ((count($elements) == 4) AND ($elements[2] == "post")) + return "http://".$server[0]."/notice/".$elements[3]; + + if ((count($conversation) != 2) OR ($conversation[1] =="")) + return $href; + + if ($elements[3] == "objectType=thread") + return "http://".$server[0]."/conversation/".$conversation[1]; + else + return "http://".$server[0]."/notice/".$conversation[1]; + + return $href; +} + +function check_conversations($override = false) { $last = get_config('system','ostatus_last_poll'); $poll_interval = intval(get_config('system','ostatus_poll_interval')); @@ -10,16 +33,16 @@ function check_conversations() { $poll_interval = OSTATUS_DEFAULT_POLL_INTERVAL; // Don't poll if the interval is set negative - if ($poll_interval < 0) + if (($poll_interval < 0) AND !$override) return; $poll_timeframe = intval(get_config('system','ostatus_poll_timeframe')); - if(! $poll_timeframe) + if (!$poll_timeframe) $poll_timeframe = OSTATUS_DEFAULT_POLL_TIMEFRAME; - if($last) { + if ($last AND !$override) { $next = $last + ($poll_interval * 60); - if($next > time()) { + if ($next > time()) { logger('poll interval not reached'); return; } @@ -36,7 +59,7 @@ function check_conversations() { complete_conversation($id, $url); } - logger(' cron_end'); + logger('cron_end'); set_config('system','ostatus_last_poll', time()); } @@ -44,6 +67,8 @@ function check_conversations() { function complete_conversation($itemid, $conversation_url, $only_add_conversation = false) { global $a; + $conversation_url = ostatus_convert_href($conversation_url); + if (intval(get_config('system','ostatus_poll_interval')) == -2) return; @@ -107,16 +132,24 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $items = array_reverse($items); foreach ($items as $single_conv) { - // status.net changed the format of the activity streams. This is a quick fix. - if (@is_string($single_conv->object->id)) + if (!isset($single_conv->id) AND isset($single_conv->object->id)) $single_conv->id = $single_conv->object->id; + elseif (!isset($single_conv->id) AND isset($single_conv->object->url)) + $single_conv->id = $single_conv->object->url; - if (@!$single_conv->id AND $single_conv->provider->url AND $single_conv->statusnet_notice_info->local_id) - $single_conv->id = $single_conv->provider->url."notice/".$single_conv->statusnet_notice_info->local_id; + $plink = ostatus_convert_href($single_conv->id); + + if (isset($single_conv->provider->url) AND isset($single_conv->statusnet_notice_info->local_id)) + $plink = $single_conv->provider->url."notice/".$single_conv->statusnet_notice_info->local_id; + elseif (isset($single_conv->provider->url) AND isset($single_conv->statusnet->notice_info->local_id)) + $plink = $single_conv->provider->url."notice/".$single_conv->statusnet->notice_info->local_id; if (@!$single_conv->id) continue; + //logger("OStatus conversation id ".$single_conv->id, LOGGER_DEBUG); + //logger("OStatus conversation data ".print_r($single_conv, true), LOGGER_DEBUG); + if ($first_id == "") { $first_id = $single_conv->id; @@ -136,8 +169,13 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio else $parent_uri = $parent["uri"]; - $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' LIMIT 1", + $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `plink` = '%s' LIMIT 1", + intval($message["uid"]), dbesc($plink)); + + if (!$message_exists) + $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' LIMIT 1", intval($message["uid"]), dbesc($single_conv->id)); + if ($message_exists) { if ($parent["id"] != 0) { $existing_message = $message_exists[0]; @@ -166,7 +204,7 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $arr = array(); $arr["network"] = NETWORK_OSTATUS; $arr["uri"] = $single_conv->id; - $arr["plink"] = $single_conv->id; + $arr["plink"] = $plink; $arr["uid"] = $message["uid"]; $arr["contact-id"] = $contact_id; if ($parent["id"] != 0) @@ -190,9 +228,16 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $arr["author-link"] = $single_conv->actor->id; $arr["author-avatar"] = $single_conv->actor->image->url; $arr["body"] = html2bbcode($single_conv->content); - $arr["app"] = strip_tags($single_conv->statusnet_notice_info->source); - if ($arr["app"] == "") + + if (isset($single_conv->statusnet->notice_info->source)) + $arr["app"] = strip_tags($single_conv->statusnet->notice_info->source); + elseif (isset($single_conv->statusnet_notice_info->source)) + $arr["app"] = strip_tags($single_conv->statusnet_notice_info->source); + elseif (isset($single_conv->provider->displayName)) $arr["app"] = $single_conv->provider->displayName; + else + $arr["app"] = "OStatus"; + $arr["verb"] = $parent["verb"]; $arr["visible"] = $parent["visible"]; $arr["location"] = $single_conv->location->displayName; @@ -206,6 +251,8 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $newitem = item_store($arr); + logger('Stored new item '.$plink.' under id '.$newitem, LOGGER_DEBUG); + // Add the conversation entry (but don't fetch the whole conversation) complete_conversation($newitem, $conversation_url, true); From bdd5c05668fafbadbc19026b75269ff9e97dbc70 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 27 May 2015 08:16:05 +0200 Subject: [PATCH 4/6] OStatus: Don't be confused by posts from the statusnet connector. --- include/items.php | 52 +++++++++++++++++++++++--------- include/ostatus_conversation.php | 23 +++++++++----- 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/include/items.php b/include/items.php index 3b4a5a193d..69b4a0f259 100644 --- a/include/items.php +++ b/include/items.php @@ -1149,6 +1149,19 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa } } + // If there is no guid then take the same guid that was taken before for the same plink + if ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) { + logger('item_store: checking for an existing guid for plink '.$arr['plink'], LOGGER_DEBUG); + $r = q("SELECT `guid` FROM `item` WHERE `plink` = '%s' AND `guid` != '' LIMIT 1", + dbesc(trim($arr['plink'])) + ); + + if(count($r)) { + $arr['guid'] = $r[0]["guid"]; + logger('item_store: found guid '.$arr['guid'].' for plink '.$arr['plink'], LOGGER_DEBUG); + } + } + // Shouldn't happen but we want to make absolutely sure it doesn't leak from a plugin. // Deactivated, since the bbcode parser can handle with it - and it destroys posts with some smileys that contain "<" //if((strpos($arr['body'],'<') !== false) || (strpos($arr['body'],'>') !== false)) @@ -1355,26 +1368,37 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa } } - $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1", + $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `network` = '%s' AND `uid` = %d LIMIT 1", dbesc($arr['uri']), + dbesc($arr['network']), intval($arr['uid']) ); - if($r && count($r)) { logger('duplicated item with the same uri found. ' . print_r($arr,true)); return 0; - } else { - // Check for an existing post with the same content. There seems to be a problem with OStatus. - $r = q("SELECT `id` FROM `item` WHERE `body` = '%s' AND `created` = '%s' AND `contact-id` = %d AND `uid` = %d LIMIT 1", - dbesc($arr['body']), - dbesc($arr['created']), - intval($arr['contact-id']), - intval($arr['uid']) - ); - if($r && count($r)) { - logger('duplicated item with the same body found. ' . print_r($arr,true)); - return 0; - } + } + + $r = q("SELECT `id` FROM `item` WHERE `plink` = '%s' AND `network` = '%s' AND `uid` = %d LIMIT 1", + dbesc($arr['plink']), + dbesc($arr['network']), + intval($arr['uid']) + ); + if($r && count($r)) { + logger('duplicated item with the same plink found. ' . print_r($arr,true)); + return 0; + } + + // Check for an existing post with the same content. There seems to be a problem with OStatus. + $r = q("SELECT `id` FROM `item` WHERE `body` = '%s' AND `network` = '%s' AND `created` = '%s' AND `contact-id` = %d AND `uid` = %d LIMIT 1", + dbesc($arr['body']), + dbesc($arr['network']), + dbesc($arr['created']), + intval($arr['contact-id']), + intval($arr['uid']) + ); + if($r && count($r)) { + logger('duplicated item with the same body found. ' . print_r($arr,true)); + return 0; } // Is this item available in the global items (with uid=0)? diff --git a/include/ostatus_conversation.php b/include/ostatus_conversation.php index da8dda292d..f21f8248f8 100644 --- a/include/ostatus_conversation.php +++ b/include/ostatus_conversation.php @@ -143,6 +143,8 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $plink = $single_conv->provider->url."notice/".$single_conv->statusnet_notice_info->local_id; elseif (isset($single_conv->provider->url) AND isset($single_conv->statusnet->notice_info->local_id)) $plink = $single_conv->provider->url."notice/".$single_conv->statusnet->notice_info->local_id; + elseif (isset($single_conv->provider->url) AND isset($single_conv->status_net->notice_info->local_id)) + $plink = $single_conv->provider->url."notice/".$single_conv->status_net->notice_info->local_id; if (@!$single_conv->id) continue; @@ -153,8 +155,9 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio if ($first_id == "") { $first_id = $single_conv->id; - $new_parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `uri` = '%s' LIMIT 1", - intval($message["uid"]), dbesc($first_id)); + $new_parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", + intval($message["uid"]), dbesc($first_id), + dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); if ($new_parents) { $parent = $new_parents[0]; logger('adopting new parent '.$parent["id"].' for '.$itemid); @@ -169,17 +172,21 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio else $parent_uri = $parent["uri"]; - $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `plink` = '%s' LIMIT 1", - intval($message["uid"]), dbesc($plink)); + $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `plink` = '%s' AND `network` IN ('%s','%s') LIMIT 1", + intval($message["uid"]), dbesc($plink), + dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); if (!$message_exists) - $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' LIMIT 1", - intval($message["uid"]), dbesc($single_conv->id)); + $message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", + intval($message["uid"]), dbesc($single_conv->id), + dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); if ($message_exists) { if ($parent["id"] != 0) { $existing_message = $message_exists[0]; + logger('updating id '.$existing_message["id"].' to parent '.$parent["id"].' uri '.$parent["uri"].' thread '.$parent_uri, LOGGER_DEBUG); + // This is partly bad, since the entry in the thread table isn't updated $r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `thr-parent` = '%s' WHERE `id` = %d", intval($parent["id"]), @@ -229,7 +236,9 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $arr["author-avatar"] = $single_conv->actor->image->url; $arr["body"] = html2bbcode($single_conv->content); - if (isset($single_conv->statusnet->notice_info->source)) + if (isset($single_conv->status_net->notice_info->source)) + $arr["app"] = strip_tags($single_conv->status_net->notice_info->source); + elseif (isset($single_conv->statusnet->notice_info->source)) $arr["app"] = strip_tags($single_conv->statusnet->notice_info->source); elseif (isset($single_conv->statusnet_notice_info->source)) $arr["app"] = strip_tags($single_conv->statusnet_notice_info->source); From 24962f300229ffd54f88e3ec7ce58573603ed573 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 28 May 2015 07:51:12 +0200 Subject: [PATCH 5/6] OStatus completion: The id and the plink for fetched items should now have reliable values. --- include/ostatus_conversation.php | 50 +++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/include/ostatus_conversation.php b/include/ostatus_conversation.php index f21f8248f8..a03330a820 100644 --- a/include/ostatus_conversation.php +++ b/include/ostatus_conversation.php @@ -1,4 +1,6 @@ id) AND isset($single_conv->object->id)) + if (isset($single_conv->object->id)) $single_conv->id = $single_conv->object->id; - elseif (!isset($single_conv->id) AND isset($single_conv->object->url)) - $single_conv->id = $single_conv->object->url; + + logger("Got id ".$single_conv->id, LOGGER_DEBUG); + + //if (!isset($single_conv->id) AND isset($single_conv->object->id)) + // $single_conv->id = $single_conv->object->id; + //elseif (!isset($single_conv->id) AND isset($single_conv->object->url)) + // $single_conv->id = $single_conv->object->url; $plink = ostatus_convert_href($single_conv->id); + if (isset($single_conv->object->url)) + $plink = ostatus_convert_href($single_conv->object->url); - if (isset($single_conv->provider->url) AND isset($single_conv->statusnet_notice_info->local_id)) - $plink = $single_conv->provider->url."notice/".$single_conv->statusnet_notice_info->local_id; - elseif (isset($single_conv->provider->url) AND isset($single_conv->statusnet->notice_info->local_id)) - $plink = $single_conv->provider->url."notice/".$single_conv->statusnet->notice_info->local_id; - elseif (isset($single_conv->provider->url) AND isset($single_conv->status_net->notice_info->local_id)) - $plink = $single_conv->provider->url."notice/".$single_conv->status_net->notice_info->local_id; + logger("Got url ".$plink, LOGGER_DEBUG); + + //if (isset($single_conv->provider->url) AND isset($single_conv->statusnet_notice_info->local_id)) + // $plink = $single_conv->provider->url."notice/".$single_conv->statusnet_notice_info->local_id; + //elseif (isset($single_conv->provider->url) AND isset($single_conv->statusnet->notice_info->local_id)) + // $plink = $single_conv->provider->url."notice/".$single_conv->statusnet->notice_info->local_id; + //elseif (isset($single_conv->provider->url) AND isset($single_conv->status_net->notice_info->local_id)) + // $plink = $single_conv->provider->url."notice/".$single_conv->status_net->notice_info->local_id; if (@!$single_conv->id) continue; @@ -197,14 +208,25 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio continue; } + $actor = $single_conv->actor->id; + if (isset($single_conv->actor->url)) + $actor = $single_conv->actor->url; + $contact = q("SELECT `id` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'", - $message["uid"], normalise_link($single_conv->actor->id), NETWORK_STATUSNET); + $message["uid"], normalise_link($actor), NETWORK_STATUSNET); if (count($contact)) { - logger("Found contact for url ".$single_conv->actor->id, LOGGER_DEBUG); + logger("Found contact for url ".$actor, LOGGER_DEBUG); $contact_id = $contact[0]["id"]; } else { - logger("No contact found for url ".$single_conv->actor->id, LOGGER_DEBUG); + logger("No contact found for url ".$actor, LOGGER_DEBUG); + + // Adding a global contact + // To-Do: Use this data for the post + $global_contact_id = get_contact($actor, 0); + + logger("Global contact ".$global_contact_id." found for url ".$actor, LOGGER_DEBUG); + $contact_id = $parent["contact-id"]; } @@ -227,12 +249,12 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio if ($arr["owner-name"] == '') $arr["owner-name"] = $single_conv->actor->displayName; - $arr["owner-link"] = $single_conv->actor->id; + $arr["owner-link"] = $actor; $arr["owner-avatar"] = $single_conv->actor->image->url; //$arr["author-name"] = $single_conv->actor->contact->displayName; //$arr["author-name"] = $single_conv->actor->contact->preferredUsername; $arr["author-name"] = $arr["owner-name"]; - $arr["author-link"] = $single_conv->actor->id; + $arr["author-link"] = $actor; $arr["author-avatar"] = $single_conv->actor->image->url; $arr["body"] = html2bbcode($single_conv->content); From 487d57df4569d2be1200f0e03c0b85bdc0ee0bfe Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 28 May 2015 08:02:34 +0200 Subject: [PATCH 6/6] Fetch the author url and not the id. --- include/items.php | 30 +++++++++++++++++++++++++++--- include/ostatus_conversation.php | 15 --------------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/include/items.php b/include/items.php index 69b4a0f259..ef164bc2ce 100644 --- a/include/items.php +++ b/include/items.php @@ -464,12 +464,27 @@ function get_atom_elements($feed, $item, $contact = array()) { // look for a photo. We should check media size and find the best one, // but for now let's just find any author photo + // Additionally we look for an alternate author link. On OStatus this one is the one we want. + + // Search for ostatus conversation url + $authorlinks = $item->feed->data["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["feed"][0]["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["author"][0]["child"]["http://www.w3.org/2005/Atom"]["link"]; + if (is_array($authorlinks)) { + foreach ($authorlinks as $link) { + $linkdata = array_shift($link["attribs"]); + + if ($linkdata["rel"] == "alternate") + $res["author-link"] = $linkdata["href"]; + }; + } $rawauthor = $item->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_10,'author'); if($rawauthor && $rawauthor[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link']) { $base = $rawauthor[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link']; foreach($base as $link) { + if($link['attribs']['']['rel'] === 'alternate') + $res['author-link'] = unxmlify($link['attribs']['']['href']); + if(!x($res, 'author-avatar') || !$res['author-avatar']) { if($link['attribs']['']['rel'] === 'photo' || $link['attribs']['']['rel'] === 'avatar') $res['author-avatar'] = unxmlify($link['attribs']['']['href']); @@ -828,7 +843,7 @@ function get_atom_elements($feed, $item, $contact = array()) { logger('get_atom_elements: Looking for status.net repeated message'); $message = $child["http://activitystrea.ms/spec/1.0/"]["object"][0]["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["content"][0]["data"]; - $orig_uri = $child["http://activitystrea.ms/spec/1.0/"]["object"][0]["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["id"][0]["data"]; + $orig_id = ostatus_convert_href($child["http://activitystrea.ms/spec/1.0/"]["object"][0]["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["id"][0]["data"]); $author = $child[SIMPLEPIE_NAMESPACE_ATOM_10]["author"][0]["child"][SIMPLEPIE_NAMESPACE_ATOM_10]; $uri = $author["uri"][0]["data"]; $name = $author["name"][0]["data"]; @@ -836,10 +851,10 @@ function get_atom_elements($feed, $item, $contact = array()) { $avatar = $avatar["href"]; if (($name != "") and ($uri != "") and ($avatar != "") and ($message != "")) { - logger('get_atom_elements: fixing sender of repeated message.'); + logger('get_atom_elements: fixing sender of repeated message. '.$orig_id, LOGGER_DEBUG); if (!intval(get_config('system','wall-to-wall_share'))) { - $prefix = share_header($name, $uri, $avatar, "", "", $orig_uri); + $prefix = share_header($name, $uri, $avatar, "", "", $orig_link); $res["body"] = $prefix.html2bbcode($message)."[/share]"; } else { @@ -866,6 +881,9 @@ function get_atom_elements($feed, $item, $contact = array()) { if ($conversation["rel"] == "ostatus:conversation") { $res["ostatus_conversation"] = ostatus_convert_href($conversation["href"]); logger('get_atom_elements: found conversation url '.$res["ostatus_conversation"]); + } elseif ($conversation["rel"] == "alternate") { + $res["plink"] = $conversation["href"]; + logger('get_atom_elements: found plink '.$res["plink"]); } }; } @@ -2212,6 +2230,12 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0) return; } + // Test - remove before flight +// if ($contact['network'] === NETWORK_OSTATUS) { +// $tempfile = tempnam(get_temppath(), "ostatus"); +// file_put_contents($tempfile, $xml); +// } + $feed = new SimplePie(); $feed->set_raw_data($xml); if($datedir) diff --git a/include/ostatus_conversation.php b/include/ostatus_conversation.php index a03330a820..f40c4827b9 100644 --- a/include/ostatus_conversation.php +++ b/include/ostatus_conversation.php @@ -139,30 +139,15 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio logger("Got id ".$single_conv->id, LOGGER_DEBUG); - //if (!isset($single_conv->id) AND isset($single_conv->object->id)) - // $single_conv->id = $single_conv->object->id; - //elseif (!isset($single_conv->id) AND isset($single_conv->object->url)) - // $single_conv->id = $single_conv->object->url; - $plink = ostatus_convert_href($single_conv->id); if (isset($single_conv->object->url)) $plink = ostatus_convert_href($single_conv->object->url); logger("Got url ".$plink, LOGGER_DEBUG); - //if (isset($single_conv->provider->url) AND isset($single_conv->statusnet_notice_info->local_id)) - // $plink = $single_conv->provider->url."notice/".$single_conv->statusnet_notice_info->local_id; - //elseif (isset($single_conv->provider->url) AND isset($single_conv->statusnet->notice_info->local_id)) - // $plink = $single_conv->provider->url."notice/".$single_conv->statusnet->notice_info->local_id; - //elseif (isset($single_conv->provider->url) AND isset($single_conv->status_net->notice_info->local_id)) - // $plink = $single_conv->provider->url."notice/".$single_conv->status_net->notice_info->local_id; - if (@!$single_conv->id) continue; - //logger("OStatus conversation id ".$single_conv->id, LOGGER_DEBUG); - //logger("OStatus conversation data ".print_r($single_conv, true), LOGGER_DEBUG); - if ($first_id == "") { $first_id = $single_conv->id;