Merge pull request #5328 from annando/item-activities

Item activities are now stored in a separate table

boot.php

@@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'The Tazmans Flax-lily');
 define('FRIENDICA_VERSION',      '2018.08-dev');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
-define('DB_UPDATE_VERSION',      1275);
+define('DB_UPDATE_VERSION',      1276);
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
 /**

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2018.08-dev (The Tazmans Flax-lily)
--- DB_UPDATE_VERSION 1274
+-- DB_UPDATE_VERSION 1276
 -- ------------------------------------------
 
 
@@ -478,6 +478,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 	`author-link` varchar(255) NOT NULL DEFAULT '' COMMENT 'Link to the profile page of the author of this item',
 	`author-avatar` varchar(255) NOT NULL DEFAULT '' COMMENT 'Link to the avatar picture of the author of this item',
 	`icid` int unsigned COMMENT 'Id of the item-content table entry that contains the whole item content',
+	`iaid` int unsigned COMMENT 'Id of the item-activity table entry that contains the activity data',
 	`title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
 	`content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`body` mediumtext COMMENT 'item body content',
@@ -539,9 +540,23 @@ CREATE TABLE IF NOT EXISTS `item` (
 	 INDEX `deleted_changed` (`deleted`,`changed`),
 	 INDEX `uid_wall_changed` (`uid`,`wall`,`changed`),
 	 INDEX `uid_eventid` (`uid`,`event-id`),
-	 INDEX `icid` (`icid`)
+	 INDEX `icid` (`icid`),
+	 INDEX `iaid` (`iaid`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
 
+--
+-- TABLE item-activity
+--
+CREATE TABLE IF NOT EXISTS `item-activity` (
+	`id` int unsigned NOT NULL auto_increment,
+	`uri` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`uri-hash` char(80) NOT NULL DEFAULT '' COMMENT 'SHA-1 and RIPEMD-160 hash from uri',
+	`activity` smallint unsigned NOT NULL DEFAULT 0 COMMENT '',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `uri-hash` (`uri-hash`),
+	 INDEX `uri` (`uri`(191))
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Activities for items';
+
 --
 -- TABLE item-content
 --

include/api.php

@@ -1278,7 +1278,7 @@ function api_status_show($type)
 	// get last public wall message
 	$condition = ['owner-id' => $user_info['pid'], 'uid' => api_user(),
 		'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]];
-	$lastwall = dba::selectFirst('item', [], $condition, ['order' => ['id' => true]]);
+	$lastwall = Item::selectFirst(Item::ITEM_FIELDLIST, $condition, ['order' => ['id' => true]]);
 
 	if (DBM::is_result($lastwall)) {
 		$in_reply_to = api_in_reply_to($lastwall);
@@ -1363,7 +1363,7 @@ function api_users_show($type)
 
 	$condition = ['owner-id' => $user_info['pid'], 'uid' => api_user(),
 		'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT], 'private' => false];
-	$lastwall = dba::selectFirst('item', [], $condition, ['order' => ['id' => true]]);
+	$lastwall = Item::selectFirst(Item::ITEM_FIELDLIST, $condition, ['order' => ['id' => true]]);
 
 	if (DBM::is_result($lastwall)) {
 		$in_reply_to = api_in_reply_to($lastwall);
@@ -1823,12 +1823,12 @@ function api_statuses_show($type)
 	$conversation = !empty($_REQUEST['conversation']);
 
 	// try to fetch the item for the local user - or the public item, if there is no local one
-	$uri_item = dba::selectFirst('item', ['uri'], ['id' => $id]);
+	$uri_item = Item::selectFirst(['uri'], ['id' => $id]);
 	if (!DBM::is_result($uri_item)) {
 		throw new BadRequestException("There is no status with this id.");
 	}
 
-	$item = dba::selectFirst('item', ['id'], ['uri' => $uri_item['uri'], 'uid' => [0, api_user()]], ['order' => ['uid' => true]]);
+	$item = Item::selectFirst(['id'], ['uri' => $uri_item['uri'], 'uid' => [0, api_user()]], ['order' => ['uid' => true]]);
 	if (!DBM::is_result($item)) {
 		throw new BadRequestException("There is no status with this id.");
 	}
@@ -1903,12 +1903,12 @@ function api_conversation_show($type)
 	logger('API: api_conversation_show: '.$id);
 
 	// try to fetch the item for the local user - or the public item, if there is no local one
-	$item = dba::selectFirst('item', ['parent-uri'], ['id' => $id]);
+	$item = Item::selectFirst(['parent-uri'], ['id' => $id]);
 	if (!DBM::is_result($item)) {
 		throw new BadRequestException("There is no status with this id.");
 	}
 
-	$parent = dba::selectFirst('item', ['id'], ['uri' => $item['parent-uri'], 'uid' => [0, api_user()]], ['order' => ['uid' => true]]);
+	$parent = Item::selectFirst(['id'], ['uri' => $item['parent-uri'], 'uid' => [0, api_user()]], ['order' => ['uid' => true]]);
 	if (!DBM::is_result($parent)) {
 		throw new BadRequestException("There is no status with this id.");
 	}

include/dba.php

@@ -955,6 +955,8 @@ class dba {
 	 * @return boolean Was the command executed successfully?
 	 */
 	public static function rollback() {
+		$ret = false;
+
 		switch (self::$driver) {
 			case 'pdo':
 				if (!self::$db->inTransaction()) {

mod/acl.php

@@ -241,7 +241,7 @@ function acl_content(App $a)
 
 	if ($conv_id) {
 		// In multi threaded posts the conv_id is not the parent of the whole thread
-		$parent_item = dba::selectFirst('item', ['parent'], ['id' => $conv_id]);
+		$parent_item = Item::selectFirst(['parent'], ['id' => $conv_id]);
 		if (DBM::is_result($parent_item)) {
 			$conv_id = $parent_item['parent'];
 		}

mod/display.php

@@ -202,7 +202,7 @@ function display_content(App $a, $update = false, $update_uid = 0)
 
 	if ($update) {
 		$item_id = $_REQUEST['item_id'];
-		$item = dba::selectFirst('item', ['uid', 'parent', 'parent-uri'], ['id' => $item_id]);
+		$item = Item::selectFirst(['uid', 'parent', 'parent-uri'], ['id' => $item_id]);
 		if ($item['uid'] != 0) {
 			$a->profile = ['uid' => intval($item['uid']), 'profile_uid' => intval($item['uid'])];
 		} else {

mod/item.php

@@ -106,7 +106,7 @@ function item_post(App $a) {
 			$thr_parent_contact = Contact::getDetailsByURL($parent_item["author-link"]);
 
 			if ($parent_item['id'] != $parent_item['parent']) {
-				$parent_item = dba::selectFirst('item', [], ['id' => $parent_item['parent']]);
+				$parent_item = Item::selectFirst(Item::ITEM_FIELDLIST, ['id' => $parent_item['parent']]);
 			}
 		}
 
@@ -170,7 +170,7 @@ function item_post(App $a) {
 	$orig_post = null;
 
 	if ($post_id) {
-		$orig_post = dba::selectFirst('item', [], ['id' => $post_id]);
+		$orig_post = Item::selectFirst(Item::ITEM_FIELDLIST, ['id' => $post_id]);
 	}
 
 	$user = dba::selectFirst('user', [], ['uid' => $profile_uid]);

src/App.php

@@ -412,11 +412,17 @@ class App
 	public function set_baseurl($url)
 	{
 		$parsed = @parse_url($url);
+		$hostname = '';
 
 		if (x($parsed)) {
+			if (!empty($parsed['scheme'])) {
 				$this->scheme = $parsed['scheme'];
+			}
 
+			if (!empty($parsed['host'])) {
 				$hostname = $parsed['host'];
+			}
+
 			if (x($parsed, 'port')) {
 				$hostname .= ':' . $parsed['port'];
 			}

src/Database/DBStructure.php

@@ -1181,6 +1181,7 @@ class DBStructure
 						"author-link" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the profile page of the author of this item"],
 						"author-avatar" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the avatar picture of the author of this item"],
 						"icid" => ["type" => "int unsigned", "relation" => ["item-content" => "id"], "comment" => "Id of the item-content table entry that contains the whole item content"],
+						"iaid" => ["type" => "int unsigned", "relation" => ["item-activity" => "id"], "comment" => "Id of the item-activity table entry that contains the activity data"],
 						"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
 						"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
 						"body" => ["type" => "mediumtext", "comment" => "item body content"],
@@ -1245,6 +1246,21 @@ class DBStructure
 						"uid_wall_changed" => ["uid","wall","changed"],
 						"uid_eventid" => ["uid","event-id"],
 						"icid" => ["icid"],
+						"iaid" => ["iaid"],
+						]
+				];
+		$database["item-activity"] = [
+				"comment" => "Activities for items",
+				"fields" => [
+						"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]],
+						"uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
+						"uri-hash" => ["type" => "char(80)", "not null" => "1", "default" => "", "comment" => "SHA-1 and RIPEMD-160 hash from uri"],
+						"activity" => ["type" => "smallint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
+						],
+				"indexes" => [
+						"PRIMARY" => ["id"],
+						"uri-hash" => ["UNIQUE", "uri-hash"],
+						"uri" => ["uri(191)"],
 						]
 				];
 		$database["item-content"] = [

src/Database/PostUpdate.php

@@ -34,6 +34,9 @@ class PostUpdate
 		if (!self::update1274()) {
 			return;
 		}
+		if (!self::update1275()) {
+			return;
+		}
 	}
 
 	/**
@@ -272,6 +275,45 @@ class PostUpdate
 		}
 		dba::close($items);
 
+		logger("Processed rows: " . $rows, LOGGER_DEBUG);
+		return true;
+	}
+	/**
+	 * @brief update the "item-activity" table
+	 *
+	 * @return bool "true" when the job is done
+	 */
+	private static function update1275()
+	{
+		// Was the script completed?
+		if (Config::get("system", "post_update_version") >= 1275) {
+			return true;
+		}
+
+		logger("Start", LOGGER_DEBUG);
+
+		$fields = ['id', 'verb'];
+
+		$condition = ["`iaid` IS NULL AND NOT `icid` IS NULL AND `verb` IN (?, ?, ?, ?, ?)",
+			ACTIVITY_LIKE, ACTIVITY_DISLIKE, ACTIVITY_ATTEND, ACTIVITY_ATTENDNO, ACTIVITY_ATTENDMAYBE];
+
+		$params = ['limit' => 10000];
+		$items = Item::select($fields, $condition, $params);
+
+		if (!DBM::is_result($items)) {
+			Config::set("system", "post_update_version", 1275);
+			logger("Done", LOGGER_DEBUG);
+			return true;
+		}
+
+		$rows = 0;
+
+		while ($item = Item::fetch($items)) {
+			Item::update($item, ['id' => $item['id']]);
+			++$rows;
+		}
+		dba::close($items);
+
 		logger("Processed rows: " . $rows, LOGGER_DEBUG);
 		return true;
 	}

src/Model/Item.php

@@ -63,7 +63,7 @@ class Item extends BaseObject
 
 	// All fields in the item table
 	const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid',
-			'contact-id', 'type', 'wall', 'gravity', 'extid', 'icid',
+			'contact-id', 'type', 'wall', 'gravity', 'extid', 'icid', 'iaid',
 			'created', 'edited', 'commented', 'received', 'changed', 'verb',
 			'postopts', 'plink', 'resource-id', 'event-id', 'tag', 'attach', 'inform',
 			'file', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
@@ -74,6 +74,42 @@ class Item extends BaseObject
 			'author-id', 'author-link', 'author-name', 'author-avatar',
 			'owner-id', 'owner-link', 'owner-name', 'owner-avatar'];
 
+	// Never reorder or remove entries from this list. Just add new ones at the end, if needed.
+	// The item-activity table only stores the index and needs this array to know the matching activity.
+	const ACTIVITIES = [ACTIVITY_LIKE, ACTIVITY_DISLIKE, ACTIVITY_ATTEND, ACTIVITY_ATTENDNO, ACTIVITY_ATTENDMAYBE];
+
+	/**
+	 * @brief returns an activity index from an activity string
+	 *
+	 * @param string $activity activity string
+	 * @return integer Activity index
+	 */
+	private static function activityToIndex($activity)
+	{
+		$index = array_search($activity, self::ACTIVITIES);
+
+		if (is_bool($index)) {
+			$index = -1;
+		}
+
+		return $index;
+	}
+
+	/**
+	 * @brief returns an activity string from an activity index
+	 *
+	 * @param integer $index activity index
+	 * @return string Activity string
+	 */
+	private static function indexToActivity($index)
+	{
+		if (is_null($index) || !array_key_exists($index, self::ACTIVITIES)) {
+			return '';
+		}
+
+		return self::ACTIVITIES[$index];
+	}
+
 	/**
 	 * @brief Fetch a single item row
 	 *
@@ -84,14 +120,12 @@ class Item extends BaseObject
 	{
 		$row = dba::fetch($stmt);
 
-		// Fetch data from the item-content table whenever there is content there
+		if (is_bool($row)) {
-		foreach (self::MIXED_CONTENT_FIELDLIST as $field) {
+			return $row;
-			if (empty($row[$field]) && !empty($row['item-' . $field])) {
-				$row[$field] = $row['item-' . $field];
-			}
-			unset($row['item-' . $field]);
 		}
 
+		// ---------------------- Transform item structure data ----------------------
+
 		// We prefer the data from the user's contact over the public one
 		if (!empty($row['author-link']) && !empty($row['contact-link']) &&
 			($row['author-link'] == $row['contact-link'])) {
@@ -113,21 +147,66 @@ class Item extends BaseObject
 			}
 		}
 
+		// We can always comment on posts from these networks
+		if (array_key_exists('writable', $row) &&
+			in_array($row['internal-network'], [NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS])) {
+			$row['writable'] = true;
+		}
+
+		// ---------------------- Transform item content data ----------------------
+
+		// Fetch data from the item-content table whenever there is content there
+		foreach (self::MIXED_CONTENT_FIELDLIST as $field) {
+			if (empty($row[$field]) && !empty($row['internal-item-' . $field])) {
+				$row[$field] = $row['internal-item-' . $field];
+			}
+			unset($row['internal-item-' . $field]);
+		}
+
+		if (!empty($row['internal-iaid']) && array_key_exists('verb', $row)) {
+			$row['verb'] = self::indexToActivity($row['internal-activity']);
+			if (array_key_exists('title', $row)) {
+				$row['title'] = '';
+			}
+			if (array_key_exists('body', $row)) {
+				$row['body'] = $row['verb'];
+			}
+			if (array_key_exists('object', $row)) {
+				$row['object'] = '';
+			}
+			if (array_key_exists('object-type', $row)) {
+				$row['object-type'] = ACTIVITY_OBJ_NOTE;
+			}
+		} elseif (array_key_exists('verb', $row) && in_array($row['verb'], ['', ACTIVITY_POST, ACTIVITY_SHARE])) {
+			// Posts don't have an object or target - but having tags or files.
+			// We safe some performance by building tag and file strings only here.
+			// We remove object and target since they aren't used for this type.
+			if (array_key_exists('object', $row)) {
+				$row['object'] = '';
+			}
+			if (array_key_exists('target', $row)) {
+				$row['target'] = '';
+			}
+		}
+
+		if (!array_key_exists('verb', $row) || in_array($row['verb'], ['', ACTIVITY_POST, ACTIVITY_SHARE])) {
 			// Build the tag string out of the term entries
-		if (isset($row['id']) && array_key_exists('tag', $row)) {
+			if (array_key_exists('tag', $row) && empty($row['tag'])) {
-			$row['tag'] = Term::tagTextFromItemId($row['id']);
+				$row['tag'] = Term::tagTextFromItemId($row['internal-iid']);
 			}
 
 			// Build the file string out of the term entries
-		if (isset($row['id']) && array_key_exists('file', $row)) {
+			if (array_key_exists('file', $row) && empty($row['file'])) {
-			$row['file'] = Term::fileTextFromItemId($row['id']);
+				$row['file'] = Term::fileTextFromItemId($row['internal-iid']);
+			}
 		}
 
-		// We can always comment on posts from these networks
+		// Remove internal fields
-		if (isset($row['writable']) && !empty($row['network']) &&
+		unset($row['internal-activity']);
-			in_array($row['network'], [NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS])) {
+		unset($row['internal-network']);
-			$row['writable'] = true;
+		unset($row['internal-iid']);
-		}
+		unset($row['internal-iaid']);
+		unset($row['internal-icid']);
 
 		return $row;
 	}
@@ -413,7 +492,11 @@ class Item extends BaseObject
 			'file', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
 			'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark',
 			'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global',
-			'id' => 'item_id', 'network', 'icid'];
+			'id' => 'item_id', 'network', 'icid', 'iaid', 'id' => 'internal-iid',
+			'network' => 'internal-network', 'icid' => 'internal-icid',
+			'iaid' => 'internal-iaid'];
+
+		$fields['item-activity'] = ['activity', 'activity' => 'internal-activity'];
 
 		$fields['item-content'] = array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST);
 
@@ -518,6 +601,10 @@ class Item extends BaseObject
 			$joins .= " LEFT JOIN `sign` ON `sign`.`iid` = `item`.`id`";
 		}
 
+		if (strpos($sql_commands, "`item-activity`.") !== false) {
+			$joins .= " LEFT JOIN `item-activity` ON `item-activity`.`id` = `item`.`iaid`";
+		}
+
 		if (strpos($sql_commands, "`item-content`.") !== false) {
 			$joins .= " LEFT JOIN `item-content` ON `item-content`.`id` = `item`.`icid`";
 		}
@@ -543,14 +630,15 @@ class Item extends BaseObject
 	 */
 	private static function constructSelectFields($fields, $selected)
 	{
-		// To be able to fetch the tags we need the item id
+		if (!empty($selected)) {
-		if (in_array('tag', $selected) && !in_array('id', $selected)) {
+			$selected[] = 'internal-iid';
-			$selected[] = 'id';
+			$selected[] = 'internal-iaid';
+			$selected[] = 'internal-icid';
+			$selected[] = 'internal-network';
 		}
 
-		// To be able to fetch the files we need the item id
+		if (in_array('verb', $selected)) {
-		if (in_array('file', $selected) && !in_array('id', $selected)) {
+			$selected[] = 'internal-activity';
-			$selected[] = 'id';
 		}
 
 		$selection = [];
@@ -558,7 +646,7 @@ class Item extends BaseObject
 			foreach ($table_fields as $field => $select) {
 				if (empty($selected) || in_array($select, $selected)) {
 					if (in_array($select, self::MIXED_CONTENT_FIELDLIST)) {
-						$selection[] = "`item`.`".$select."` AS `item-" . $select . "`";
+						$selection[] = "`item`.`".$select."` AS `internal-item-" . $select . "`";
 					}
 					if (is_int($field)) {
 						$selection[] = "`" . $table . "`.`" . $select . "`";
@@ -622,13 +710,24 @@ class Item extends BaseObject
 		// We cannot simply expand the condition to check for origin entries
 		// The condition needn't to be a simple array but could be a complex condition.
 		// And we have to execute this query before the update to ensure to fetch the same data.
-		$items = dba::select('item', ['id', 'origin', 'uri', 'plink', 'icid'], $condition);
+		$items = dba::select('item', ['id', 'origin', 'uri', 'plink', 'iaid', 'icid', 'tag', 'file'], $condition);
 
 		$content_fields = [];
 		foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
 			if (isset($fields[$field])) {
 				$content_fields[$field] = $fields[$field];
+				if (in_array($field, self::CONTENT_FIELDLIST)) {
 					unset($fields[$field]);
+				} else {
+					$fields[$field] = null;
+				}
+			}
+		}
+
+		$author_owner_fields = ['author-name', 'author-avatar', 'author-link', 'owner-name', 'owner-avatar', 'owner-link'];
+		foreach ($author_owner_fields as $field) {
+			if (isset($fields[$field])) {
+				$fields[$field] = null;
 			}
 		}
 
@@ -663,6 +762,30 @@ class Item extends BaseObject
 			if (!empty($item['plink'])) {
 				$content_fields['plink'] = $item['plink'];
 			}
+			if (!empty($item['iaid']) || (!empty($content_fields['verb']) && (self::activityToIndex($content_fields['verb']) >= 0))) {
+				self::updateActivity($content_fields, ['uri' => $item['uri']]);
+
+				if (empty($item['iaid'])) {
+					$item_activity = dba::selectFirst('item-activity', ['id'], ['uri' => $item['uri']]);
+					if (DBM::is_result($item_activity)) {
+						$item_fields = ['iaid' => $item_activity['id'], 'icid' => null];
+						foreach (self::MIXED_CONTENT_FIELDLIST as $field) {
+							$item_fields[$field] = '';
+						}
+						dba::update('item', $item_fields, ['id' => $item['id']]);
+
+						if (!empty($item['icid']) && !dba::exists('item', ['icid' => $item['icid']])) {
+							dba::delete('item-content', ['id' => $item['icid']]);
+						}
+					}
+				} elseif (!empty($item['icid'])) {
+					dba::update('item', ['icid' => null], ['id' => $item['id']]);
+
+					if (!dba::exists('item', ['icid' => $item['icid']])) {
+						dba::delete('item-content', ['id' => $item['icid']]);
+					}
+				}
+			} else {
 				self::updateContent($content_fields, ['uri' => $item['uri']]);
 
 				if (empty($item['icid'])) {
@@ -678,13 +801,20 @@ class Item extends BaseObject
 						dba::update('item', $item_fields, ['id' => $item['id']]);
 					}
 				}
+			}
 
 			if (!empty($tags)) {
 				Term::insertFromTagFieldByItemId($item['id'], $tags);
+				if (!empty($item['tag'])) {
+					dba::update('item', ['tag' => ''], ['id' => $item['id']]);
+				}
 			}
 
 			if (!empty($files)) {
 				Term::insertFromFileFieldByItemId($item['id'], $files);
+				if (!empty($item['file'])) {
+					dba::update('item', ['file' => ''], ['id' => $item['id']]);
+				}
 			}
 
 			self::updateThread($item['id']);
@@ -1420,7 +1550,9 @@ class Item extends BaseObject
 		}
 
 		// We are doing this outside of the transaction to avoid timing problems
+		if (!self::insertActivity($item)) {
 			self::insertContent($item);
+		}
 
 		dba::transaction();
 		$ret = dba::insert('item', $item);
@@ -1575,6 +1707,55 @@ class Item extends BaseObject
 		return $current_post;
 	}
 
+	/**
+	 * @brief Insert a new item content entry
+	 *
+	 * @param array $item The item fields that are to be inserted
+	 */
+	private static function insertActivity(&$item)
+	{
+		$activity_index = self::activityToIndex($item['verb']);
+
+		if ($activity_index < 0) {
+			return false;
+		}
+
+		$fields = ['uri' => $item['uri'], 'activity' => $activity_index,
+			'uri-hash' => hash('sha1', $item['uri']) . hash('ripemd160', $item['uri'])];
+
+		$saved_item = $item;
+
+		// We just remove everything that is content
+		foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
+			unset($item[$field]);
+		}
+
+		// To avoid timing problems, we are using locks.
+		$locked = Lock::set('item_insert_activity');
+		if (!$locked) {
+			logger("Couldn't acquire lock for URI " . $item['uri'] . " - proceeding anyway.");
+		}
+
+		// Do we already have this content?
+		$item_activity = dba::selectFirst('item-activity', ['id'], ['uri' => $item['uri']]);
+		if (DBM::is_result($item_activity)) {
+			$item['iaid'] = $item_activity['id'];
+			logger('Fetched activity for URI ' . $item['uri'] . ' (' . $item['iaid'] . ')');
+		} elseif (dba::insert('item-activity', $fields)) {
+			$item['iaid'] = dba::lastInsertId();
+			logger('Inserted activity for URI ' . $item['uri'] . ' (' . $item['iaid'] . ')');
+		} else {
+			// This shouldn't happen. But if it does, we simply store it in the item-content table
+			logger('Could not insert activity for URI ' . $item['uri'] . ' - should not happen');
+			$item = $saved_item;
+			return false;
+		}
+		if ($locked) {
+			Lock::remove('item_insert_activity');
+		}
+		return true;
+	}
+
 	/**
 	 * @brief Insert a new item content entry
 	 *
@@ -1633,6 +1814,33 @@ class Item extends BaseObject
 		}
 	}
 
+	/**
+	 * @brief Update existing item content entries
+	 *
+	 * @param array $item The item fields that are to be changed
+	 * @param array $condition The condition for finding the item content entries
+	 */
+	private static function updateActivity($item, $condition)
+	{
+		if (empty($item['verb'])) {
+			return false;
+		}
+		$activity_index = self::activityToIndex($item['verb']);
+
+		if ($activity_index < 0) {
+			return false;
+		}
+
+		$fields = ['activity' => $activity_index,
+			'uri-hash' => hash('sha1', $condition['uri']) . hash('ripemd160', $condition['uri'])];
+
+		logger('Update activity for URI ' . $condition['uri']);
+
+		dba::update('item-activity', $fields, $condition, true);
+
+		return true;
+	}
+
 	/**
 	 * @brief Update existing item content entries
 	 *
@@ -1650,7 +1858,9 @@ class Item extends BaseObject
 		}
 
 		if (empty($fields)) {
-			return;
+			// when there are no fields at all, just use the condition
+			// This is to ensure that we always store content.
+			$fields = $condition;
 		}
 
 		if (!empty($item['plink'])) {
@@ -2605,27 +2815,22 @@ class Item extends BaseObject
 		switch ($verb) {
 			case 'like':
 			case 'unlike':
-				$bodyverb = L10n::t('%1$s likes %2$s\'s %3$s');
 				$activity = ACTIVITY_LIKE;
 				break;
 			case 'dislike':
 			case 'undislike':
-				$bodyverb = L10n::t('%1$s doesn\'t like %2$s\'s %3$s');
 				$activity = ACTIVITY_DISLIKE;
 				break;
 			case 'attendyes':
 			case 'unattendyes':
-				$bodyverb = L10n::t('%1$s is attending %2$s\'s %3$s');
 				$activity = ACTIVITY_ATTEND;
 				break;
 			case 'attendno':
 			case 'unattendno':
-				$bodyverb = L10n::t('%1$s is not attending %2$s\'s %3$s');
 				$activity = ACTIVITY_ATTENDNO;
 				break;
 			case 'attendmaybe':
 			case 'unattendmaybe':
-				$bodyverb = L10n::t('%1$s may attend %2$s\'s %3$s');
 				$activity = ACTIVITY_ATTENDMAYBE;
 				break;
 			default:
@@ -2644,6 +2849,8 @@ class Item extends BaseObject
 			return false;
 		}
 
+		$item_uri = $item['uri'];
+
 		$uid = $item['uid'];
 		if (($uid == 0) && local_user()) {
 			$uid = local_user();
@@ -2664,7 +2871,7 @@ class Item extends BaseObject
 		// Retrieve the current logged in user's public contact
 		$author_id = public_contact();
 
-		$author_contact = dba::selectFirst('contact', [], ['id' => $author_id]);
+		$author_contact = dba::selectFirst('contact', ['url'], ['id' => $author_id]);
 		if (!DBM::is_result($author_contact)) {
 			logger('like: unknown author ' . $author_id);
 			return false;
@@ -2688,26 +2895,21 @@ class Item extends BaseObject
 		// we need to eradicate your first choice.
 		if ($event_verb_flag) {
 			$verbs = [ACTIVITY_ATTEND, ACTIVITY_ATTENDNO, ACTIVITY_ATTENDMAYBE];
+
+			// Translate to the index based activity index
+			$activities = [];
+			foreach ($verbs as $verb) {
+				$activities[] = self::activityToIndex($verb);
+			}
 		} else {
-			$verbs = $activity;
+			$activities = self::activityToIndex($activity);
 		}
 
-		$base_condition = ['verb' => $verbs, 'deleted' => false, 'gravity' => GRAVITY_ACTIVITY,
+		$condition = ['activity' => $activities, 'deleted' => false, 'gravity' => GRAVITY_ACTIVITY,
-			'author-id' => $author_contact['id'], 'uid' => $item['uid']];
+			'author-id' => $author_id, 'uid' => $item['uid'], 'thr-parent' => $item_uri];
 
-		$condition = array_merge($base_condition, ['parent' => $item_id]);
 		$like_item = self::selectFirst(['id', 'guid', 'verb'], $condition);
 
-		if (!DBM::is_result($like_item)) {
-			$condition = array_merge($base_condition, ['parent-uri' => $item_id]);
-			$like_item = self::selectFirst(['id', 'guid', 'verb'], $condition);
-		}
-
-		if (!DBM::is_result($like_item)) {
-			$condition = array_merge($base_condition, ['thr-parent' => $item_id]);
-			$like_item = self::selectFirst(['id', 'guid', 'verb'], $condition);
-		}
-
 		// If it exists, mark it as deleted
 		if (DBM::is_result($like_item)) {
 			// Already voted, undo it
@@ -2716,12 +2918,9 @@ class Item extends BaseObject
 			dba::update('item', $fields, ['id' => $like_item['id']]);
 
 			// Clean up the Diaspora signatures for this like
-			// Go ahead and do it even if Diaspora support is disabled. We still want to clean up
-			// if it had been enabled in the past
 			dba::delete('sign', ['iid' => $like_item['id']]);
 
-			$like_item_id = $like_item['id'];
+			Worker::add(PRIORITY_HIGH, "Notifier", "like", $like_item['id']);
-			Worker::add(PRIORITY_HIGH, "Notifier", "like", $like_item_id);
 
 			if (!$event_verb_flag || $like_item['verb'] == $activity) {
 				return true;
@@ -2733,30 +2932,7 @@ class Item extends BaseObject
 			return true;
 		}
 
-		// Else or if event verb different from existing row, create a new item row
-		$post_type = (($item['resource-id']) ? L10n::t('photo') : L10n::t('status'));
-		if ($item['object-type'] === ACTIVITY_OBJ_EVENT) {
-			$post_type = L10n::t('event');
-		}
 		$objtype = $item['resource-id'] ? ACTIVITY_OBJ_IMAGE : ACTIVITY_OBJ_NOTE ;
-		$link = xmlify('<link rel="alternate" type="text/html" href="' . System::baseUrl() . '/display/' . $owner_self_contact['nick'] . '/' . $item['id'] . '" />' . "\n") ;
-		$body = $item['body'];
-
-		$obj = <<< EOT
-
-		<object>
-			<type>$objtype</type>
-			<local>1</local>
-			<id>{$item['uri']}</id>
-			<link>$link</link>
-			<title></title>
-			<content>$body</content>
-		</object>
-EOT;
-
-		$ulink = '[url=' . $author_contact['url'] . ']' . $author_contact['name'] . '[/url]';
-		$alink = '[url=' . $item['author-link'] . ']' . $item['author-name'] . '[/url]';
-		$plink = '[url=' . System::baseUrl() . '/display/' . $owner_self_contact['nick'] . '/' . $item['id'] . ']' . $post_type . '[/url]';
 
 		$new_item = [
 			'guid'          => get_guid(32),
@@ -2771,17 +2947,10 @@ EOT;
 			'parent-uri'    => $item['uri'],
 			'thr-parent'    => $item['uri'],
 			'owner-id'      => $item['owner-id'],
-			'owner-name'    => $item['owner-name'],
+			'author-id'     => $author_id,
-			'owner-link'    => $item['owner-link'],
+			'body'          => $activity,
-			'owner-avatar'  => $item['owner-avatar'],
-			'author-id'     => $author_contact['id'],
-			'author-name'   => $author_contact['name'],
-			'author-link'   => $author_contact['url'],
-			'author-avatar' => $author_contact['thumb'],
-			'body'          => sprintf($bodyverb, $ulink, $alink, $plink),
 			'verb'          => $activity,
 			'object-type'   => $objtype,
-			'object'        => $obj,
 			'allow_cid'     => $item['allow_cid'],
 			'allow_gid'     => $item['allow_gid'],
 			'deny_cid'      => $item['deny_cid'],

src/Object/Image.php

@@ -781,6 +781,7 @@ class Image
 			$img_str = Network::fetchUrl($url, true, $redirects, 4);
 			$filesize = strlen($img_str);
 
+			try {
 				if (function_exists("getimagesizefromstring")) {
 					$data = getimagesizefromstring($img_str);
 				} else {
@@ -794,6 +795,9 @@ class Image
 					$data = getimagesize($tempfile);
 					unlink($tempfile);
 				}
+			} catch (Exception $e) {
+				return false;
+			}
 
 			if ($data) {
 				$data["size"] = $filesize;

src/Object/Post.php

@@ -13,6 +13,7 @@ use Friendica\Core\L10n;
 use Friendica\Core\PConfig;
 use Friendica\Database\DBM;
 use Friendica\Model\Contact;
+use Friendica\Model\Item;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Temporal;
 use dba;
@@ -178,7 +179,7 @@ class Post extends BaseObject
 		if (!$origin) {
 			/// @todo This shouldn't be done as query here, but better during the data creation.
 			// it is now done here, since during the RC phase we shouldn't make to intense changes.
-			$parent = dba::selectFirst('item', ['origin'], ['id' => $item['parent']]);
+			$parent = Item::selectFirst(['origin'], ['id' => $item['parent']]);
 			if (DBM::is_result($parent)) {
 				$origin = $parent['origin'];
 			}

src/Protocol/DFRN.php

@@ -2414,8 +2414,7 @@ class DFRN
 
 		$item["edited"] = $xpath->query("atom:updated/text()", $entry)->item(0)->nodeValue;
 
-		$current = dba::selectFirst('item',
+		$current = Item::selectFirst(['id', 'uid', 'edited', 'body'],
-			['id', 'uid', 'edited', 'body'],
 			['uri' => $item["uri"], 'uid' => $importer["importer_uid"]]
 		);
 		// Is there an existing item?
@@ -2747,13 +2746,18 @@ class DFRN
 			return false;
 		}
 
-		$condition = ["`uri` = ? AND `uid` = ? AND NOT `file` LIKE '%[%'", $uri, $importer["importer_uid"]];
+		$condition = ['uri' => $uri, 'uid' => $importer["importer_uid"]];
-		$item = dba::selectFirst('item', ['id', 'parent', 'contact-id'], $condition);
+		$item = Item::selectFirst(['id', 'parent', 'contact-id', 'file'], $condition);
 		if (!DBM::is_result($item)) {
 			logger("Item with uri " . $uri . " for user " . $importer["importer_uid"] . " wasn't found.", LOGGER_DEBUG);
 			return;
 		}
 
+		if (strstr($item['file'], '[')) {
+			logger("Item with uri " . $uri . " for user " . $importer["importer_uid"] . " is filed. So it won't be deleted.", LOGGER_DEBUG);
+			return;
+		}
+
 		// When it is a starting post it has to belong to the person that wants to delete it
 		if (($item['id'] == $item['parent']) && ($item['contact-id'] != $importer["id"])) {
 			logger("Item with uri " . $uri . " don't belong to contact " . $importer["id"] . " - ignoring deletion.", LOGGER_DEBUG);

src/Protocol/Diaspora.php

@@ -1924,50 +1924,6 @@ class Diaspora
 		return true;
 	}
 
-	/**
-	 * @brief Creates the body for a "like" message
-	 *
-	 * @param array  $contact     The contact that send us the "like"
-	 * @param array  $parent_item The item array of the parent item
-	 * @param string $guid        message guid
-	 *
-	 * @return string the body
-	 */
-	private static function constructLikeBody($contact, $parent_item, $guid)
-	{
-		$bodyverb = L10n::t('%1$s likes %2$s\'s %3$s');
-
-		$ulink = "[url=".$contact["url"]."]".$contact["name"]."[/url]";
-		$alink = "[url=".$parent_item["author-link"]."]".$parent_item["author-name"]."[/url]";
-		$plink = "[url=".System::baseUrl()."/display/".urlencode($guid)."]".L10n::t("status")."[/url]";
-
-		return sprintf($bodyverb, $ulink, $alink, $plink);
-	}
-
-	/**
-	 * @brief Creates a XML object for a "like"
-	 *
-	 * @param array $importer    Array of the importer user
-	 * @param array $parent_item The item array of the parent item
-	 *
-	 * @return string The XML
-	 */
-	private static function constructLikeObject($importer, $parent_item)
-	{
-		$objtype = ACTIVITY_OBJ_NOTE;
-		$link = '<link rel="alternate" type="text/html" href="'.System::baseUrl()."/display/".$importer["nickname"]."/".$parent_item["id"].'" />';
-		$parent_body = $parent_item["body"];
-
-		$xmldata = ["object" => ["type" => $objtype,
-						"local" => "1",
-						"id" => $parent_item["uri"],
-						"link" => $link,
-						"title" => "",
-						"content" => $parent_body]];
-
-		return XML::fromArray($xmldata, $xml, true);
-	}
-
 	/**
 	 * @brief Processes "like" messages
 	 *
@@ -2046,9 +2002,8 @@ class Diaspora
 		$datarray["parent-uri"] = $parent_item["uri"];
 
 		$datarray["object-type"] = ACTIVITY_OBJ_NOTE;
-		$datarray["object"] = self::constructLikeObject($importer, $parent_item);
 
-		$datarray["body"] = self::constructLikeBody($contact, $parent_item, $guid);
+		$datarray["body"] = $verb;
 
 		// like on comments have the comment as parent. So we need to fetch the toplevel parent
 		if ($parent_item["id"] != $parent_item["parent"]) {
@@ -2750,14 +2705,15 @@ class Diaspora
 		}
 
 		// Fetch items that are about to be deleted
-		$fields = ['uid', 'id', 'parent', 'parent-uri', 'author-link'];
+		$fields = ['uid', 'id', 'parent', 'parent-uri', 'author-link', 'file'];
 
 		// When we receive a public retraction, we delete every item that we find.
 		if ($importer['uid'] == 0) {
-			$condition = ["`guid` = ? AND NOT `file` LIKE '%%[%%' AND NOT `deleted`", $target_guid];
+			$condition = ['guid' => $target_guid, 'deleted' => false];
 		} else {
-			$condition = ["`guid` = ? AND `uid` = ? AND NOT `file` LIKE '%%[%%' AND NOT `deleted`", $target_guid, $importer['uid']];
+			$condition = ['guid' => $target_guid, 'deleted' => false, 'uid' => $importer['uid']];
 		}
+
 		$r = Item::select($fields, $condition);
 		if (!DBM::is_result($r)) {
 			logger("Target guid ".$target_guid." was not found on this system for user ".$importer['uid'].".");
@@ -2765,6 +2721,11 @@ class Diaspora
 		}
 
 		while ($item = Item::fetch($r)) {
+			if (strstr($item['file'], '[')) {
+				logger("Target guid " . $target_guid . " for user " . $item['uid'] . " is filed. So it won't be deleted.", LOGGER_DEBUG);
+				continue;
+			}
+
 			// Fetch the parent item
 			$parent = Item::selectFirst(['author-link'], ['id' => $item["parent"]]);
 

src/Protocol/Feed.php

@@ -250,7 +250,7 @@ class Feed {
 			if (!$simulate) {
 				$condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)",
 					$importer["uid"], $item["uri"], NETWORK_FEED, NETWORK_DFRN];
-				$previous = dba::selectFirst('item', ['id'], $condition);
+				$previous = Item::selectFirst(['id'], $condition);
 				if (DBM::is_result($previous)) {
 					logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$previous["id"], LOGGER_DEBUG);
 					continue;

src/Protocol/OStatus.php

@@ -338,7 +338,7 @@ class OStatus
 		$header = [];
 		$header["uid"] = $importer["uid"];
 		$header["network"] = NETWORK_OSTATUS;
-		$header["type"] = "remote";
+		$header["type"] = "remote-comment";
 		$header["wall"] = 0;
 		$header["origin"] = 0;
 		$header["gravity"] = GRAVITY_COMMENT;
@@ -449,9 +449,11 @@ class OStatus
 				$orig_uri = $xpath->query("activity:object/atom:id", $entry)->item(0)->nodeValue;
 				logger("Favorite ".$orig_uri." ".print_r($item, true));
 
+				$item["type"] = "activity";
 				$item["verb"] = ACTIVITY_LIKE;
 				$item["parent-uri"] = $orig_uri;
 				$item["gravity"] = GRAVITY_ACTIVITY;
+				$item["object-type"] = ACTIVITY_OBJ_NOTE;
 			}
 
 			// http://activitystrea.ms/schema/1.0/rsvp-yes
@@ -681,11 +683,10 @@ class OStatus
 			} else {
 				logger('Reply with URI '.$item["uri"].' already existed for user '.$importer["uid"].'.', LOGGER_DEBUG);
 			}
-
-			$item["type"] = 'remote-comment';
 		} else {
 			$item["parent-uri"] = $item["uri"];
 			$item["gravity"] = GRAVITY_PARENT;
+			$item["type"] = "remote";
 		}
 
 		if (($item['author-link'] != '') && !empty($item['protocol'])) {

src/Worker/Delivery.php

@@ -53,7 +53,7 @@ class Delivery extends BaseObject
 		} elseif ($cmd == self::RELOCATION) {
 			$uid = $item_id;
 		} else {
-			$item = dba::selectFirst('item', ['parent'], ['id' => $item_id]);
+			$item = Item::selectFirst(['parent'], ['id' => $item_id]);
 			if (!DBM::is_result($item) || empty($item['parent'])) {
 				return;
 			}
@@ -436,12 +436,12 @@ class Delivery extends BaseObject
 
 			if (empty($target_item['title'])) {
 				$condition = ['uri' => $target_item['parent-uri'], 'uid' => $owner['uid']];
-				$title = dba::selectFirst('item', ['title'], $condition);
+				$title = Item::selectFirst(['title'], $condition);
 				if (DBM::is_result($title) && ($title['title'] != '')) {
 					$subject = $title['title'];
 				} else {
 					$condition = ['parent-uri' => $target_item['parent-uri'], 'uid' => $owner['uid']];
-					$title = dba::selectFirst('item', ['title'], $condition);
+					$title = Item::selectFirst(['title'], $condition);
 					if (DBM::is_result($title) && ($title['title'] != '')) {
 						$subject = $title['title'];
 					}

src/Worker/Expire.php

@@ -26,21 +26,37 @@ class Expire {
 		if ($param == 'delete') {
 			logger('Delete expired items', LOGGER_DEBUG);
 			// physically remove anything that has been deleted for more than two months
-			$r = dba::p("SELECT `id`, `icid` FROM `item` WHERE `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY");
+			$condition = ["`deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY"];
-			while ($row = dba::fetch($r)) {
+			$rows = dba::select('item', ['id', 'iaid', 'icid'],  $condition);
+			while ($row = dba::fetch($rows)) {
 				dba::delete('item', ['id' => $row['id']]);
-				if (!dba::exists('item', ['icid' => $row['icid']])) {
+				if (!empty($row['iaid']) && !dba::exists('item', ['iaid' => $row['iaid']])) {
+					dba::delete('item-activity', ['id' => $row['iaid']]);
+				}
+				if (!empty($row['icid']) && !dba::exists('item', ['icid' => $row['icid']])) {
 					dba::delete('item-content', ['id' => $row['icid']]);
 				}
 			}
-			dba::close($r);
+			dba::close($rows);
 
-			logger('Delete expired items - done', LOGGER_DEBUG);
+			// Normally we shouldn't have orphaned data at all.
+			// If we do have some, then we have to check why.
+			logger('Deleting orphaned item activities - start', LOGGER_DEBUG);
+			$condition = ["NOT EXISTS (SELECT `iaid` FROM `item` WHERE `item`.`uri` = `item-activity`.`uri`)"];
+			dba::delete('item-activity', $condition);
+			logger('Orphaned item activities deleted: ' . dba::affected_rows(), LOGGER_DEBUG);
+
+			logger('Deleting orphaned item content - start', LOGGER_DEBUG);
+			$condition = ["NOT EXISTS (SELECT `icid` FROM `item` WHERE `item`.`uri` = `item-content`.`uri`)"];
+			dba::delete('item-content', $condition);
+			logger('Orphaned item content deleted: ' . dba::affected_rows(), LOGGER_DEBUG);
 
 			// make this optional as it could have a performance impact on large sites
 			if (intval(Config::get('system', 'optimize_items'))) {
 				dba::e("OPTIMIZE TABLE `item`");
 			}
+
+			logger('Delete expired items - done', LOGGER_DEBUG);
 			return;
 		} elseif (intval($param) > 0) {
 			$user = dba::selectFirst('user', ['uid', 'username', 'expire'], ['uid' => $param]);

src/Worker/Notifier.php

@@ -167,7 +167,7 @@ class Notifier {
 
 			$fields = ['network', 'author-id', 'owner-id'];
 			$condition = ['uri' => $target_item["thr-parent"], 'uid' => $target_item["uid"]];
-			$thr_parent = dba::selectFirst('item', $fields, $condition);
+			$thr_parent = Item::selectFirst($fields, $condition);
 
 			logger('GUID: '.$target_item["guid"].': Parent is '.$parent['network'].'. Thread parent is '.$thr_parent['network'], LOGGER_DEBUG);