From 8d4736c942b9ccfc3473264684d6ccd1081c428c Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 Sep 2017 08:01:22 +0000 Subject: [PATCH 1/2] OStatus: We can now process delete messages / better account detection --- boot.php | 1 + include/Contact.php | 26 ++++++++- include/ostatus.php | 132 +++++++++++++++++++++++++++++--------------- include/threads.php | 5 +- 4 files changed, 115 insertions(+), 49 deletions(-) diff --git a/boot.php b/boot.php index 417502b960..32fee4f509 100644 --- a/boot.php +++ b/boot.php @@ -394,6 +394,7 @@ define ( 'ACTIVITY_POST', NAMESPACE_ACTIVITY_SCHEMA . 'post' ); define ( 'ACTIVITY_UPDATE', NAMESPACE_ACTIVITY_SCHEMA . 'update' ); define ( 'ACTIVITY_TAG', NAMESPACE_ACTIVITY_SCHEMA . 'tag' ); define ( 'ACTIVITY_FAVORITE', NAMESPACE_ACTIVITY_SCHEMA . 'favorite' ); +define ( 'ACTIVITY_UNFAVORITE', NAMESPACE_ACTIVITY_SCHEMA . 'unfavorite' ); define ( 'ACTIVITY_SHARE', NAMESPACE_ACTIVITY_SCHEMA . 'share' ); define ( 'ACTIVITY_DELETE', NAMESPACE_ACTIVITY_SCHEMA . 'delete' ); diff --git a/include/Contact.php b/include/Contact.php index 69c3b121f8..a4477c8cb3 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -206,6 +206,8 @@ function get_contact_details_by_url($url, $uid = -1, $default = array()) { return $cache[$url][$uid]; } + $ssl_url = str_replace('http://', 'https://', $url); + // Fetch contact data from the contact table for the given user $s = dba::p("SELECT `id`, `id` AS `cid`, 0 AS `gid`, 0 AS `zid`, `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, `xmpp`, `keywords`, `gender`, `photo`, `thumb`, `micro`, `forum`, `prv`, (`forum` | `prv`) AS `community`, `contact-type`, `bd` AS `birthday`, `self` @@ -213,6 +215,15 @@ function get_contact_details_by_url($url, $uid = -1, $default = array()) { normalise_link($url), $uid); $r = dba::inArray($s); + // Fetch contact data from the contact table for the given user, checking with the alias + if (!dbm::is_result($r)) { + $s = dba::p("SELECT `id`, `id` AS `cid`, 0 AS `gid`, 0 AS `zid`, `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, `xmpp`, + `keywords`, `gender`, `photo`, `thumb`, `micro`, `forum`, `prv`, (`forum` | `prv`) AS `community`, `contact-type`, `bd` AS `birthday`, `self` + FROM `contact` WHERE `alias` IN (?, ?, ?) AND `uid` = ?", + normalise_link($url), $url, $ssl_url, $uid); + $r = dba::inArray($s); + } + // Fetch the data from the contact table with "uid=0" (which is filled automatically) if (!dbm::is_result($r)) { $s = dba::p("SELECT `id`, 0 AS `cid`, `id` AS `zid`, 0 AS `gid`, `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, `xmpp`, @@ -222,6 +233,15 @@ function get_contact_details_by_url($url, $uid = -1, $default = array()) { $r = dba::inArray($s); } + // Fetch the data from the contact table with "uid=0" (which is filled automatically) - checked with the alias + if (!dbm::is_result($r)) { + $s = dba::p("SELECT `id`, 0 AS `cid`, `id` AS `zid`, 0 AS `gid`, `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, `xmpp`, + `keywords`, `gender`, `photo`, `thumb`, `micro`, `forum`, `prv`, (`forum` | `prv`) AS `community`, `contact-type`, `bd` AS `birthday`, 0 AS `self` + FROM `contact` WHERE `alias` IN (?, ?, ?) AND `uid` = 0", + normalise_link($url), $url, $ssl_url); + $r = dba::inArray($s); + } + // Fetch the data from the gcontact table if (!dbm::is_result($r)) { $s = dba::p("SELECT 0 AS `id`, 0 AS `cid`, `id` AS `gid`, 0 AS `zid`, 0 AS `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, '' AS `xmpp`, @@ -542,8 +562,10 @@ function get_contact($url, $uid = 0, $no_update = false) { // Then the alias (which could be anything) if (!dbm::is_result($contact)) { - $r = dba::p("SELECT `id`, `avatar-date` FROM `contact` WHERE `alias` IN (?, ?) AND `uid` = ? LIMIT 1", - $url, normalise_link($url), $uid); + // The link could be provided as http although we stored it as https + $ssl_url = str_replace('http://', 'https://', $url); + $r = dba::p("SELECT `id`, `avatar-date` FROM `contact` WHERE `alias` IN (?, ?, ?) AND `uid` = ? LIMIT 1", + $url, normalise_link($url), $ssl_url, $uid); $contact = dba::fetch($r); dba::close($r); } diff --git a/include/ostatus.php b/include/ostatus.php index 45ac8061c9..9472c56e03 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -7,6 +7,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; use Friendica\Network\Probe; +use Friendica\Util\Lock; require_once 'include/Contact.php'; require_once 'include/threads.php'; @@ -64,28 +65,34 @@ class ostatus { $author["contact-id"] = $contact["id"]; + $found = false; + if ($author["author-link"] != "") { if ($aliaslink == "") { $aliaslink = $author["author-link"]; } - $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `nurl` IN ('%s', '%s') AND `network` != '%s'", - intval($importer["uid"]), dbesc(normalise_link($author["author-link"])), - dbesc(normalise_link($aliaslink)), dbesc(NETWORK_STATUSNET)); + $condition = array("`uid` = ? AND `nurl` IN (?, ?) AND `network` != ?", $importer["uid"], + normalise_link($author["author-link"]), normalise_link($aliaslink), NETWORK_STATUSNET); + $r = dba::select('contact', array(), $condition, array('limit' => 1)); if (dbm::is_result($r)) { - $contact = $r[0]; - $author["contact-id"] = $r[0]["id"]; - $author["author-link"] = $r[0]["url"]; + $found = true; + $contact = $r; + $author["contact-id"] = $r["id"]; + $author["author-link"] = $r["url"]; } - } elseif ($addr != "") { - // Should not happen - $contact = dba::fetch_first("SELECT * FROM `contact` WHERE `uid` = ? AND `addr` = ? AND `network` != ?", - $importer["uid"], $addr, NETWORK_STATUSNET); + } - if (dbm::is_result($contact)) { - $author["contact-id"] = $contact["id"]; - $author["author-link"] = $contact["url"]; + if (!$found && ($addr != "")) { + $condition = array("`uid` = ? AND `addr` = ? AND `network` != ?", + $importer["uid"], $addr, NETWORK_STATUSNET); + $r = dba::select('contact', array(), $condition, array('limit' => 1)); + + if (dbm::is_result($r)) { + $contact = $r; + $author["contact-id"] = $r["id"]; + $author["author-link"] = $r["url"]; } } @@ -176,14 +183,16 @@ class ostatus { $cid = get_contact($author["author-link"], 0); if ($cid) { + $fields = array('url', 'name', 'nick', 'alias', 'about', 'location'); + $old_contact = dba::select('contact', $fields, array('id' => $cid), array('limit' => 1)); + // Update it with the current values - q("UPDATE `contact` SET `url` = '%s', `name` = '%s', `nick` = '%s', `alias` = '%s', - `about` = '%s', `location` = '%s', - `success_update` = '%s', `last-update` = '%s' - WHERE `id` = %d", - dbesc($author["author-link"]), dbesc($contact["name"]), dbesc($contact["nick"]), - dbesc($contact["alias"]), dbesc($contact["about"]), dbesc($contact["location"]), - dbesc(datetime_convert()), dbesc(datetime_convert()), intval($cid)); + $fields = array('url' => $author["author-link"], 'name' => $contact["name"], + 'nick' => $contact["nick"], 'alias' => $contact["alias"], + 'about' => $contact["about"], 'location' => $contact["location"], + 'success_update' => datetime_convert(), 'last-update' => datetime_convert()); + + dba::update('contact', $fields, array('id' => $cid), $old_contact); // Update the avatar update_contact_avatar($author["author-avatar"], 0, $cid); @@ -370,18 +379,39 @@ class ostatus { $item = array_merge($header, $author); + $item["uri"] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue; + $item["verb"] = $xpath->query('activity:verb/text()', $entry)->item(0)->nodeValue; - /// Delete a message + // Delete a message if (in_array($item["verb"], array('qvitter-delete-notice', ACTIVITY_DELETE, 'delete'))) { - // ignore "Delete" messages (by now) - logger("Ignore delete message ".print_r($item, true)); + self::deleteNotice($item); continue; } + if (in_array($item["verb"], array(NAMESPACE_OSTATUS."/unfavorite", ACTIVITY_UNFAVORITE))) { + // Ignore "Unfavorite" message + logger("Ignore unfavorite message ".print_r($item, true), LOGGER_DEBUG); + continue; + } + + // Deletions come with the same uri, so we check for duplicates after processing deletions + if (dba::exists('item', array('uid' => $importer["uid"], 'uri' => $item["uri"]))) { + logger('Post with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.', LOGGER_DEBUG); + continue; + } else { + logger('Processing post with URI '.$item["uri"].' for user '.$importer["uid"].'.', LOGGER_DEBUG); + } + if ($item["verb"] == ACTIVITY_JOIN) { // ignore "Join" messages - logger("Ignore join message ".print_r($item, true)); + logger("Ignore join message ".print_r($item, true), LOGGER_DEBUG); + continue; + } + + if ($item["verb"] == "http://mastodon.social/schema/1.0/block") { + // ignore mastodon "block" messages + logger("Ignore block message ".print_r($item, true), LOGGER_DEBUG); continue; } @@ -395,12 +425,6 @@ class ostatus { continue; } - if ($item["verb"] == NAMESPACE_OSTATUS."/unfavorite") { - // Ignore "Unfavorite" message - logger("Ignore unfavorite message ".print_r($item, true)); - continue; - } - if ($item["verb"] == ACTIVITY_FAVORITE) { $orig_uri = $xpath->query("activity:object/atom:id", $entry)->item(0)->nodeValue; logger("Favorite ".$orig_uri." ".print_r($item, true)); @@ -411,10 +435,8 @@ class ostatus { } // http://activitystrea.ms/schema/1.0/rsvp-yes - // http://activitystrea.ms/schema/1.0/unfavorite - // http://mastodon.social/schema/1.0/block if (!in_array($item["verb"], array(ACTIVITY_POST, ACTIVITY_LIKE, ACTIVITY_SHARE))) { - logger("Unhandled verb ".$item["verb"]." ".print_r($item, true)); + logger("Unhandled verb ".$item["verb"]." ".print_r($item, true), LOGGER_DEBUG); } self::processPost($xpath, $entry, $item, $importer); @@ -457,17 +479,44 @@ class ostatus { if ($found) { logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already exists.", LOGGER_DEBUG); } else { - $ret = item_store($item); - logger('Item was stored with return value '.$ret); + // We are having duplicated entries. Hopefully this solves it. + if (Lock::set('ostatus_process_item_store')) { + $ret = item_store($item); + Lock::remove('ostatus_process_item_store'); + logger("Item with uri ".$item["uri"]." for user ".$importer["uid"].' stored. Return value: '.$ret); + } else { + $ret = item_store($item); + logger("We couldn't lock - but tried to store the item anyway. Return value is ".$ret); + } } } } self::$itemlist = array(); } + logger('Processing done for post with URI '.$item["uri"].' for user '.$importer["uid"].'.', LOGGER_DEBUG); } return true; } + private static function deleteNotice($item) { + + $condition = array('uid' => $item['uid'], 'author-link' => $item['author-link'], 'uri' => $item['uri']); + $deleted = dba::select('item', array('id', 'parent-uri'), $condition, array('limit' => 1)); + if (!dbm::is_result($deleted)) { + logger('Item from '.$item['author-link'].' with uri '.$item['uri'].' for user '.$item['uid']." wasn't found. We don't delete it. "); + return; + } + + // Currently we don't have a central deletion function that we could use in this case. The function "item_drop" doesn't work for that case + dba::update('item', array('deleted' => true, 'title' => '', 'body' => '', + 'edited' => datetime_convert(), 'changed' => datetime_convert()), + array('id' => $deleted["id"])); + + delete_thread($deleted["id"], $deleted["parent-uri"]); + + logger('Deleted item with uri '.$item['uri'].' for user '.$item['uid']); + } + /** * @brief Processes the XML for a post * @@ -477,13 +526,6 @@ class ostatus { * @param array $importer user record of the importing user */ private static function processPost($xpath, $entry, &$item, $importer) { - $item["uri"] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue; - - if (dba::exists('item', array('uid' => $importer["uid"], 'uri' => $item["uri"]))) { - logger('Post with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.'); - return; - } - $item["body"] = html2bbcode($xpath->query('atom:content/text()', $entry)->item(0)->nodeValue); $item["object-type"] = $xpath->query('activity:object-type/text()', $entry)->item(0)->nodeValue; if (($item["object-type"] == ACTIVITY_OBJ_BOOKMARK) || ($item["object-type"] == ACTIVITY_OBJ_EVENT)) { @@ -601,7 +643,7 @@ class ostatus { if (!dba::exists('item', array('uid' => $importer["uid"], 'uri' => $item['parent-uri']))) { self::fetchRelated($related, $item["parent-uri"], $importer); } else { - logger('Reply with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.'); + logger('Reply with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.', LOGGER_DEBUG); } $item["type"] = 'remote-comment'; @@ -766,7 +808,7 @@ class ostatus { private static function fetchSelf($self, &$item) { $condition = array('`item-uri` = ? AND `protocol` IN (?, ?)', $self, PROTOCOL_DFRN, PROTOCOL_OSTATUS_SALMON); if (dba::exists('conversation', $condition)) { - logger('Conversation '.$item['uri'].' is already stored.'); + logger('Conversation '.$item['uri'].' is already stored.', LOGGER_DEBUG); return; } @@ -786,7 +828,7 @@ class ostatus { $item["protocol"] = PROTOCOL_OSTATUS_SALMON; $item["source"] = $xml; - logger('Conversation '.$item['uri'].' is now fetched.'); + logger('Conversation '.$item['uri'].' is now fetched.', LOGGER_DEBUG); } /** diff --git a/include/threads.php b/include/threads.php index 906766da87..8e58a04b31 100644 --- a/include/threads.php +++ b/include/threads.php @@ -240,18 +240,19 @@ function delete_thread_uri($itemuri, $uid) { function delete_thread($itemid, $itemuri = "") { $item = q("SELECT `uid` FROM `thread` WHERE `iid` = %d", intval($itemid)); + // Using dba::delete at this time could delete the associated item entries $result = q("DELETE FROM `thread` WHERE `iid` = %d", intval($itemid)); logger("delete_thread: Deleted thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG); if ($itemuri != "") { - $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND NOT (`uid` IN (%d, 0))", + $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND NOT `deleted` AND NOT (`uid` IN (%d, 0))", dbesc($itemuri), intval($item["uid"]) ); if (!dbm::is_result($r)) { dba::delete('item', array('uri' => $itemuri, 'uid' => 0)); - logger("delete_thread: Deleted shadow for item ".$itemuri." - ".print_r($result, true), LOGGER_DEBUG); + logger("delete_thread: Deleted shadow for item ".$itemuri, LOGGER_DEBUG); } } } From 926314a24c2b042f7a794c8daf8c78dd35549e85 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 Sep 2017 08:13:10 +0000 Subject: [PATCH 2/2] gcontact is now updated with the new database functions as well --- include/socgraph.php | 55 ++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index 621085b0ee..081290d810 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -2238,22 +2238,22 @@ function update_gcontact($contact) { if ($update) { logger("Update gcontact for ".$contact["url"], LOGGER_DEBUG); + $condition = array('`nurl` = ? AND (`generation` = 0 OR `generation` >= ?)', + normalise_link($contact["url"]), $contact["generation"]); + $contact["updated"] = dbm::date($contact["updated"]); - q("UPDATE `gcontact` SET `photo` = '%s', `name` = '%s', `nick` = '%s', `addr` = '%s', `network` = '%s', - `birthday` = '%s', `gender` = '%s', `keywords` = '%s', `hide` = %d, `nsfw` = %d, - `contact-type` = %d, `alias` = '%s', `notify` = '%s', `url` = '%s', - `location` = '%s', `about` = '%s', `generation` = %d, `updated` = '%s', - `server_url` = '%s', `connect` = '%s' - WHERE `nurl` = '%s' AND (`generation` = 0 OR `generation` >= %d)", - dbesc($contact["photo"]), dbesc($contact["name"]), dbesc($contact["nick"]), - dbesc($contact["addr"]), dbesc($contact["network"]), dbesc($contact["birthday"]), - dbesc($contact["gender"]), dbesc($contact["keywords"]), intval($contact["hide"]), - intval($contact["nsfw"]), intval($contact["contact-type"]), dbesc($contact["alias"]), - dbesc($contact["notify"]), dbesc($contact["url"]), dbesc($contact["location"]), - dbesc($contact["about"]), intval($contact["generation"]), dbesc(dbm::date($contact["updated"])), - dbesc($contact["server_url"]), dbesc($contact["connect"]), - dbesc(normalise_link($contact["url"])), intval($contact["generation"])); + $updated = array('photo' => $contact['photo'], 'name' => $contact['name'], + 'nick' => $contact['nick'], 'addr' => $contact['addr'], + 'network' => $contact['network'], 'birthday' => $contact['birthday'], + 'gender' => $contact['gender'], 'keywords' => $contact['keywords'], + 'hide' => $contact['hide'], 'nsfw' => $contact['nsfw'], + 'contact-type' => $contact['contact-type'], 'alias' => $contact['alias'], + 'notify' => $contact['notify'], 'url' => $contact['url'], + 'location' => $contact['location'], 'about' => $contact['about'], + 'generation' => $contact['generation'], 'updated' => $contact['updated'], + 'server_url' => $contact['server_url'], 'connect' => $contact['connect']); + dba::update('gcontact', $updated, $condition, $fields); // Now update the contact entry with the user id "0" as well. // This is used for the shadow copies of public items. @@ -2261,20 +2261,25 @@ function update_gcontact($contact) { dbesc(normalise_link($contact["url"]))); if (dbm::is_result($r)) { - logger("Update shadow contact ".$r[0]["id"], LOGGER_DEBUG); + logger("Update public contact ".$r[0]["id"], LOGGER_DEBUG); update_contact_avatar($contact["photo"], 0, $r[0]["id"]); - q("UPDATE `contact` SET `name` = '%s', `nick` = '%s', `addr` = '%s', - `network` = '%s', `bd` = '%s', `gender` = '%s', - `keywords` = '%s', `alias` = '%s', `contact-type` = %d, - `url` = '%s', `location` = '%s', `about` = '%s' - WHERE `id` = %d", - dbesc($contact["name"]), dbesc($contact["nick"]), dbesc($contact["addr"]), - dbesc($contact["network"]), dbesc($contact["birthday"]), dbesc($contact["gender"]), - dbesc($contact["keywords"]), dbesc($contact["alias"]), intval($contact["contact-type"]), - dbesc($contact["url"]), dbesc($contact["location"]), dbesc($contact["about"]), - intval($r[0]["id"])); + $fields = array('name', 'nick', 'addr', + 'network', 'bd', 'gender', + 'keywords', 'alias', 'contact-type', + 'url', 'location', 'about'); + $old_contact = dba::select('contact', $fields, array('id' => $r[0]["id"]), array('limit' => 1)); + + // Update it with the current values + $fields = array('name' => $contact['name'], 'nick' => $contact['nick'], + 'addr' => $contact['addr'], 'network' => $contact['network'], + 'bd' => $contact['birthday'], 'gender' => $contact['gender'], + 'keywords' => $contact['keywords'], 'alias' => $contact['alias'], + 'contact-type' => $contact['contact-type'], 'url' => $contact['url'], + 'location' => $contact['location'], 'about' => $contact['about']); + + dba::update('contact', $fields, array('id' => $r[0]["id"]), $old_contact); } }