OStatus: We can now process delete messages / better account detection

boot.php

@@ -394,6 +394,7 @@ define ( 'ACTIVITY_POST',        NAMESPACE_ACTIVITY_SCHEMA . 'post' );
 define ( 'ACTIVITY_UPDATE',      NAMESPACE_ACTIVITY_SCHEMA . 'update' );
 define ( 'ACTIVITY_TAG',         NAMESPACE_ACTIVITY_SCHEMA . 'tag' );
 define ( 'ACTIVITY_FAVORITE',    NAMESPACE_ACTIVITY_SCHEMA . 'favorite' );
+define ( 'ACTIVITY_UNFAVORITE',  NAMESPACE_ACTIVITY_SCHEMA . 'unfavorite' );
 define ( 'ACTIVITY_SHARE',       NAMESPACE_ACTIVITY_SCHEMA . 'share' );
 define ( 'ACTIVITY_DELETE',      NAMESPACE_ACTIVITY_SCHEMA . 'delete' );
 

include/Contact.php

@@ -206,6 +206,8 @@ function get_contact_details_by_url($url, $uid = -1, $default = array()) {
 		return $cache[$url][$uid];
 	}
 
+	$ssl_url = str_replace('http://', 'https://', $url);
+
 	// Fetch contact data from the contact table for the given user
 	$s = dba::p("SELECT `id`, `id` AS `cid`, 0 AS `gid`, 0 AS `zid`, `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, `xmpp`,
 			`keywords`, `gender`, `photo`, `thumb`, `micro`, `forum`, `prv`, (`forum` | `prv`) AS `community`, `contact-type`, `bd` AS `birthday`, `self`
@@ -213,6 +215,15 @@ function get_contact_details_by_url($url, $uid = -1, $default = array()) {
 			normalise_link($url), $uid);
 	$r = dba::inArray($s);
 
+	// Fetch contact data from the contact table for the given user, checking with the alias
+	if (!dbm::is_result($r)) {
+		$s = dba::p("SELECT `id`, `id` AS `cid`, 0 AS `gid`, 0 AS `zid`, `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, `xmpp`,
+				`keywords`, `gender`, `photo`, `thumb`, `micro`, `forum`, `prv`, (`forum` | `prv`) AS `community`, `contact-type`, `bd` AS `birthday`, `self`
+			FROM `contact` WHERE `alias` IN (?, ?, ?) AND `uid` = ?",
+				normalise_link($url), $url, $ssl_url, $uid);
+		$r = dba::inArray($s);
+	}
+
 	// Fetch the data from the contact table with "uid=0" (which is filled automatically)
 	if (!dbm::is_result($r)) {
 		$s = dba::p("SELECT `id`, 0 AS `cid`, `id` AS `zid`, 0 AS `gid`, `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, `xmpp`,
@@ -222,6 +233,15 @@ function get_contact_details_by_url($url, $uid = -1, $default = array()) {
 		$r = dba::inArray($s);
 	}
 
+	// Fetch the data from the contact table with "uid=0" (which is filled automatically) - checked with the alias
+	if (!dbm::is_result($r)) {
+		$s = dba::p("SELECT `id`, 0 AS `cid`, `id` AS `zid`, 0 AS `gid`, `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, `xmpp`,
+			`keywords`, `gender`, `photo`, `thumb`, `micro`, `forum`, `prv`, (`forum` | `prv`) AS `community`, `contact-type`, `bd` AS `birthday`, 0 AS `self`
+			FROM `contact` WHERE `alias` IN (?, ?, ?) AND `uid` = 0",
+				normalise_link($url), $url, $ssl_url);
+		$r = dba::inArray($s);
+	}
+
 	// Fetch the data from the gcontact table
 	if (!dbm::is_result($r)) {
 		$s = dba::p("SELECT 0 AS `id`, 0 AS `cid`, `id` AS `gid`, 0 AS `zid`, 0 AS `uid`, `url`, `nurl`, `alias`, `network`, `name`, `nick`, `addr`, `location`, `about`, '' AS `xmpp`,
@@ -542,8 +562,10 @@ function get_contact($url, $uid = 0, $no_update = false) {
 
 	// Then the alias (which could be anything)
 	if (!dbm::is_result($contact)) {
-		$r = dba::p("SELECT `id`, `avatar-date` FROM `contact` WHERE `alias` IN (?, ?) AND `uid` = ? LIMIT 1",
+		// The link could be provided as http although we stored it as https
-				$url, normalise_link($url), $uid);
+		$ssl_url = str_replace('http://', 'https://', $url);
+		$r = dba::p("SELECT `id`, `avatar-date` FROM `contact` WHERE `alias` IN (?, ?, ?) AND `uid` = ? LIMIT 1",
+				$url, normalise_link($url), $ssl_url, $uid);
 		$contact = dba::fetch($r);
 		dba::close($r);
 	}

include/ostatus.php

@@ -7,6 +7,7 @@ use Friendica\App;
 use Friendica\Core\System;
 use Friendica\Core\Config;
 use Friendica\Network\Probe;
+use Friendica\Util\Lock;
 
 require_once 'include/Contact.php';
 require_once 'include/threads.php';
@@ -64,28 +65,34 @@ class ostatus {
 
 		$author["contact-id"] = $contact["id"];
 
+		$found = false;
+
 		if ($author["author-link"] != "") {
 			if ($aliaslink == "") {
 				$aliaslink = $author["author-link"];
 			}
 
-			$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `nurl` IN ('%s', '%s') AND `network` != '%s'",
+			$condition = array("`uid` = ? AND `nurl` IN (?, ?) AND `network` != ?", $importer["uid"],
-				intval($importer["uid"]), dbesc(normalise_link($author["author-link"])),
+					normalise_link($author["author-link"]), normalise_link($aliaslink), NETWORK_STATUSNET);
-				dbesc(normalise_link($aliaslink)), dbesc(NETWORK_STATUSNET));
+			$r = dba::select('contact', array(), $condition, array('limit' => 1));
 
 			if (dbm::is_result($r)) {
-				$contact = $r[0];
+				$found = true;
-				$author["contact-id"] = $r[0]["id"];
+				$contact = $r;
-				$author["author-link"] = $r[0]["url"];
+				$author["contact-id"] = $r["id"];
+				$author["author-link"] = $r["url"];
 			}
-		} elseif ($addr != "") {
+		}
-			// Should not happen
-			$contact = dba::fetch_first("SELECT * FROM `contact` WHERE `uid` = ? AND `addr` = ? AND `network` != ?",
-					$importer["uid"], $addr, NETWORK_STATUSNET);
 
-			if (dbm::is_result($contact)) {
+		if (!$found && ($addr != "")) {
-				$author["contact-id"] = $contact["id"];
+			$condition = array("`uid` = ? AND `addr` = ? AND `network` != ?",
-				$author["author-link"] = $contact["url"];
+					$importer["uid"], $addr, NETWORK_STATUSNET);
+			$r = dba::select('contact', array(), $condition, array('limit' => 1));
+
+			if (dbm::is_result($r)) {
+				$contact = $r;
+				$author["contact-id"] = $r["id"];
+				$author["author-link"] = $r["url"];
 			}
 		}
 
@@ -176,14 +183,16 @@ class ostatus {
 			$cid = get_contact($author["author-link"], 0);
 
 			if ($cid) {
+				$fields = array('url', 'name', 'nick', 'alias', 'about', 'location');
+				$old_contact = dba::select('contact', $fields, array('id' => $cid), array('limit' => 1));
+
 				// Update it with the current values
-				q("UPDATE `contact` SET `url` = '%s', `name` = '%s', `nick` = '%s', `alias` = '%s',
+				$fields = array('url' => $author["author-link"], 'name' => $contact["name"],
-						`about` = '%s', `location` = '%s',
+						'nick' => $contact["nick"], 'alias' => $contact["alias"],
-						`success_update` = '%s', `last-update` = '%s'
+						'about' => $contact["about"], 'location' => $contact["location"],
-					WHERE `id` = %d",
+						'success_update' => datetime_convert(), 'last-update' => datetime_convert());
-					dbesc($author["author-link"]), dbesc($contact["name"]), dbesc($contact["nick"]),
+
-					dbesc($contact["alias"]), dbesc($contact["about"]), dbesc($contact["location"]),
+				dba::update('contact', $fields, array('id' => $cid), $old_contact);
-					dbesc(datetime_convert()), dbesc(datetime_convert()), intval($cid));
 
 				// Update the avatar
 				update_contact_avatar($author["author-avatar"], 0, $cid);
@@ -370,18 +379,39 @@ class ostatus {
 
 			$item = array_merge($header, $author);
 
+			$item["uri"] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue;
+
 			$item["verb"] = $xpath->query('activity:verb/text()', $entry)->item(0)->nodeValue;
 
-			/// Delete a message
+			// Delete a message
 			if (in_array($item["verb"], array('qvitter-delete-notice', ACTIVITY_DELETE, 'delete'))) {
-				// ignore "Delete" messages (by now)
+				self::deleteNotice($item);
-				logger("Ignore delete message ".print_r($item, true));
 				continue;
 			}
 
+			if (in_array($item["verb"], array(NAMESPACE_OSTATUS."/unfavorite", ACTIVITY_UNFAVORITE))) {
+				// Ignore "Unfavorite" message
+				logger("Ignore unfavorite message ".print_r($item, true), LOGGER_DEBUG);
+				continue;
+			}
+
+			// Deletions come with the same uri, so we check for duplicates after processing deletions
+			if (dba::exists('item', array('uid' => $importer["uid"], 'uri' => $item["uri"]))) {
+				logger('Post with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.', LOGGER_DEBUG);
+				continue;
+			} else {
+				logger('Processing post with URI '.$item["uri"].' for user '.$importer["uid"].'.', LOGGER_DEBUG);
+			}
+
 			if ($item["verb"] == ACTIVITY_JOIN) {
 				// ignore "Join" messages
-				logger("Ignore join message ".print_r($item, true));
+				logger("Ignore join message ".print_r($item, true), LOGGER_DEBUG);
+				continue;
+			}
+
+			if ($item["verb"] == "http://mastodon.social/schema/1.0/block") {
+				// ignore mastodon "block" messages
+				logger("Ignore block message ".print_r($item, true), LOGGER_DEBUG);
 				continue;
 			}
 
@@ -395,12 +425,6 @@ class ostatus {
 				continue;
 			}
 
-			if ($item["verb"] == NAMESPACE_OSTATUS."/unfavorite") {
-				// Ignore "Unfavorite" message
-				logger("Ignore unfavorite message ".print_r($item, true));
-				continue;
-			}
-
 			if ($item["verb"] == ACTIVITY_FAVORITE) {
 				$orig_uri = $xpath->query("activity:object/atom:id", $entry)->item(0)->nodeValue;
 				logger("Favorite ".$orig_uri." ".print_r($item, true));
@@ -411,10 +435,8 @@ class ostatus {
 			}
 
 			// http://activitystrea.ms/schema/1.0/rsvp-yes
-			// http://activitystrea.ms/schema/1.0/unfavorite
-			// http://mastodon.social/schema/1.0/block
 			if (!in_array($item["verb"], array(ACTIVITY_POST, ACTIVITY_LIKE, ACTIVITY_SHARE))) {
-				logger("Unhandled verb ".$item["verb"]." ".print_r($item, true));
+				logger("Unhandled verb ".$item["verb"]." ".print_r($item, true), LOGGER_DEBUG);
 			}
 
 			self::processPost($xpath, $entry, $item, $importer);
@@ -457,17 +479,44 @@ class ostatus {
 						if ($found) {
 							logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already exists.", LOGGER_DEBUG);
 						} else {
-							$ret = item_store($item);
+							// We are having duplicated entries. Hopefully this solves it.
-							logger('Item was stored with return value '.$ret);
+							if (Lock::set('ostatus_process_item_store')) {
+								$ret = item_store($item);
+								Lock::remove('ostatus_process_item_store');
+								logger("Item with uri ".$item["uri"]." for user ".$importer["uid"].' stored. Return value: '.$ret);
+							} else {
+								$ret = item_store($item);
+								logger("We couldn't lock - but tried to store the item anyway. Return value is ".$ret);
+							}
 						}
 					}
 				}
 				self::$itemlist = array();
 			}
+			logger('Processing done for post with URI '.$item["uri"].' for user '.$importer["uid"].'.', LOGGER_DEBUG);
 		}
 		return true;
 	}
 
+	private static function deleteNotice($item) {
+
+		$condition = array('uid' => $item['uid'], 'author-link' => $item['author-link'], 'uri' => $item['uri']);
+		$deleted = dba::select('item', array('id', 'parent-uri'), $condition, array('limit' => 1));
+		if (!dbm::is_result($deleted)) {
+			logger('Item from '.$item['author-link'].' with uri '.$item['uri'].' for user '.$item['uid']." wasn't found. We don't delete it. ");
+			return;
+		}
+
+		// Currently we don't have a central deletion function that we could use in this case. The function "item_drop" doesn't work for that case
+		dba::update('item', array('deleted' => true, 'title' => '', 'body' => '',
+					'edited' => datetime_convert(), 'changed' => datetime_convert()),
+				array('id' => $deleted["id"]));
+
+		delete_thread($deleted["id"], $deleted["parent-uri"]);
+
+		logger('Deleted item with uri '.$item['uri'].' for user '.$item['uid']);
+	}
+
 	/**
 	 * @brief Processes the XML for a post
 	 *
@@ -477,13 +526,6 @@ class ostatus {
 	 * @param array $importer user record of the importing user
 	 */
 	private static function processPost($xpath, $entry, &$item, $importer) {
-		$item["uri"] = $xpath->query('atom:id/text()', $entry)->item(0)->nodeValue;
-
-		if (dba::exists('item', array('uid' => $importer["uid"], 'uri' => $item["uri"]))) {
-			logger('Post with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.');
-			return;
-		}
-
 		$item["body"] = html2bbcode($xpath->query('atom:content/text()', $entry)->item(0)->nodeValue);
 		$item["object-type"] = $xpath->query('activity:object-type/text()', $entry)->item(0)->nodeValue;
 		if (($item["object-type"] == ACTIVITY_OBJ_BOOKMARK) || ($item["object-type"] == ACTIVITY_OBJ_EVENT)) {
@@ -601,7 +643,7 @@ class ostatus {
 			if (!dba::exists('item', array('uid' => $importer["uid"], 'uri' => $item['parent-uri']))) {
 				self::fetchRelated($related, $item["parent-uri"], $importer);
 			} else {
-				logger('Reply with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.');
+				logger('Reply with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.', LOGGER_DEBUG);
 			}
 
 			$item["type"] = 'remote-comment';
@@ -766,7 +808,7 @@ class ostatus {
 	private static function fetchSelf($self, &$item) {
 		$condition = array('`item-uri` = ? AND `protocol` IN (?, ?)', $self, PROTOCOL_DFRN, PROTOCOL_OSTATUS_SALMON);
 		if (dba::exists('conversation', $condition)) {
-			logger('Conversation '.$item['uri'].' is already stored.');
+			logger('Conversation '.$item['uri'].' is already stored.', LOGGER_DEBUG);
 			return;
 		}
 
@@ -786,7 +828,7 @@ class ostatus {
 		$item["protocol"] = PROTOCOL_OSTATUS_SALMON;
 		$item["source"] = $xml;
 
-		logger('Conversation '.$item['uri'].' is now fetched.');
+		logger('Conversation '.$item['uri'].' is now fetched.', LOGGER_DEBUG);
 	}
 
 	/**

include/threads.php

@@ -240,18 +240,19 @@ function delete_thread_uri($itemuri, $uid) {
 function delete_thread($itemid, $itemuri = "") {
 	$item = q("SELECT `uid` FROM `thread` WHERE `iid` = %d", intval($itemid));
 
+	// Using dba::delete at this time could delete the associated item entries
 	$result = q("DELETE FROM `thread` WHERE `iid` = %d", intval($itemid));
 
 	logger("delete_thread: Deleted thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG);
 
 	if ($itemuri != "") {
-		$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND NOT (`uid` IN (%d, 0))",
+		$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND NOT `deleted` AND NOT (`uid` IN (%d, 0))",
 				dbesc($itemuri),
 				intval($item["uid"])
 			);
 		if (!dbm::is_result($r)) {
 			dba::delete('item', array('uri' => $itemuri, 'uid' => 0));
-			logger("delete_thread: Deleted shadow for item ".$itemuri." - ".print_r($result, true), LOGGER_DEBUG);
+			logger("delete_thread: Deleted shadow for item ".$itemuri, LOGGER_DEBUG);
 		}
 	}
 }