From 06da3084f51261e9c5402c0a630ad9941f32189d Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 12 May 2020 20:13:48 +0000
Subject: [PATCH 1/6] Restructured item insert

---
 src/Model/Item.php       | 746 +++++++++++++++++++--------------------
 src/Protocol/OStatus.php |   4 +-
 2 files changed, 366 insertions(+), 384 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 2fc8a31fc..78dd6aff2 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1333,7 +1333,269 @@ class Item
 		}
 	}
 
-	public static function insert($item, $force_parent = false, $notify = false, $dontcache = false)
+	private static function isDuplicate($item)
+	{
+		// Checking if there is already an item with the same guid
+		$condition = ['guid' => $item['guid'], 'network' => $item['network'], 'uid' => $item['uid']];
+		if (self::exists($condition)) {
+			Logger::notice('Found already existing item', [
+				'guid' => $item['guid'],
+				'uid' => $item['uid'],
+				'network' => $item['network']
+			]);
+			return true;
+		}
+
+		$condition = ["`uri` = ? AND `network` IN (?, ?) AND `uid` = ?",
+			$item['uri'], $item['network'], Protocol::DFRN, $item['uid']];
+		if (self::exists($condition)) {
+			Logger::notice('duplicated item with the same uri found.', $item);
+			return true;
+		}
+
+		// On Friendica and Diaspora the GUID is unique
+		if (in_array($item['network'], [Protocol::DFRN, Protocol::DIASPORA])) {
+			$condition = ['guid' => $item['guid'], 'uid' => $item['uid']];
+			if (self::exists($condition)) {
+				Logger::notice('duplicated item with the same guid found.', $item);
+				return true;
+			}
+		} elseif ($item['network'] == Protocol::OSTATUS) {
+			// Check for an existing post with the same content. There seems to be a problem with OStatus.
+			$condition = ["`body` = ? AND `network` = ? AND `created` = ? AND `contact-id` = ? AND `uid` = ?",
+					$item['body'], $item['network'], $item['created'], $item['contact-id'], $item['uid']];
+			if (self::exists($condition)) {
+				Logger::notice('duplicated item with the same body found.', $item);
+				return true;
+			}
+		}
+
+		/*
+		 * Check for already added items.
+		 * There is a timing issue here that sometimes creates double postings.
+		 * An unique index would help - but the limitations of MySQL (maximum size of index values) prevent this.
+		 */
+		if (($item["uid"] == 0) && self::exists(['uri' => trim($item['uri']), 'uid' => 0])) {
+			Logger::notice('Global item already stored.', ['uri' => $item['uri'], 'network' => $item['network']]);
+			return true;
+		}
+
+		return false;
+	}
+
+	private static function validItem($item)
+	{
+		// When there is no content then we don't post it
+		if ($item['body'].$item['title'] == '') {
+			Logger::notice('No body, no title.');
+			return false;
+		}
+
+		// check for create date and expire time
+		$expire_interval = DI::config()->get('system', 'dbclean-expire-days', 0);
+
+		$user = DBA::selectFirst('user', ['expire'], ['uid' => $item['uid']]);
+		if (DBA::isResult($user) && ($user['expire'] > 0) && (($user['expire'] < $expire_interval) || ($expire_interval == 0))) {
+			$expire_interval = $user['expire'];
+		}
+
+		if (($expire_interval > 0) && !empty($item['created'])) {
+			$expire_date = time() - ($expire_interval * 86400);
+			$created_date = strtotime($item['created']);
+			if ($created_date < $expire_date) {
+				Logger::notice('Item created before expiration interval.', [
+					'created' => date('c', $created_date),
+					'expired' => date('c', $expire_date),
+					'$item' => $item
+				]);
+				return false;
+			}
+		}
+
+		if (Contact::isBlocked($item['author-id'])) {
+			Logger::notice('Author is blocked node-wide', ['author-link' => $item['author-link'], 'item-uri' => $item['uri']]);
+			return false;
+		}
+
+		if (!empty($item['author-link']) && Network::isUrlBlocked($item['author-link'])) {
+			Logger::notice('Author server is blocked', ['author-link' => $item['author-link'], 'item-uri' => $item['uri']]);
+			return false;
+		}
+
+		if (!empty($item['uid']) && Contact::isBlockedByUser($item['author-id'], $item['uid'])) {
+			Logger::notice('Author is blocked by user', ['author-link' => $item['author-link'], 'uid' => $item['uid'], 'item-uri' => $item['uri']]);
+			return false;
+		}
+
+		if (Contact::isBlocked($item['owner-id'])) {
+			Logger::notice('Owner is blocked node-wide', ['owner-link' => $item['owner-link'], 'item-uri' => $item['uri']]);
+			return false;
+		}
+
+		if (!empty($item['owner-link']) && Network::isUrlBlocked($item['owner-link'])) {
+			Logger::notice('Owner server is blocked', ['owner-link' => $item['owner-link'], 'item-uri' => $item['uri']]);
+			return false;
+		}
+
+		if (!empty($item['uid']) && Contact::isBlockedByUser($item['owner-id'], $item['uid'])) {
+			Logger::notice('Owner is blocked by user', ['owner-link' => $item['owner-link'], 'uid' => $item['uid'], 'item-uri' => $item['uri']]);
+			return false;
+		}
+
+		// The causer is set during a thread completion, for example because of a reshare. It countains the responsible actor.
+		if (!empty($item['uid']) && !empty($item['causer-id']) && Contact::isBlockedByUser($item['causer-id'], $item['uid'])) {
+			Logger::notice('Causer is blocked by user', ['causer-link' => $item['causer-link'], 'uid' => $item['uid'], 'item-uri' => $item['uri']]);
+			return false;
+		}
+
+		if (!empty($item['uid']) && !empty($item['causer-id']) && ($item['parent-uri'] == $item['uri']) && Contact::isIgnoredByUser($item['causer-id'], $item['uid'])) {
+			Logger::notice('Causer is ignored by user', ['causer-link' => $item['causer-link'], 'uid' => $item['uid'], 'item-uri' => $item['uri']]);
+			return false;
+		}
+		
+		if ($item['verb'] == Activity::FOLLOW) {
+			if (!$item['origin'] && ($item['author-id'] == Contact::getPublicIdByUserId($item['uid']))) {
+				// Our own follow request can be relayed to us. We don't store it to avoid notification chaos.
+				Logger::log("Follow: Don't store not origin follow request from us for " . $item['parent-uri'], Logger::DEBUG);
+				return false;
+			}
+
+			$condition = ['verb' => Activity::FOLLOW, 'uid' => $item['uid'],
+				'parent-uri' => $item['parent-uri'], 'author-id' => $item['author-id']];
+			if (self::exists($condition)) {
+				// It happens that we receive multiple follow requests by the same author - we only store one.
+				Logger::log('Follow: Found existing follow request from author ' . $item['author-id'] . ' for ' . $item['parent-uri'], Logger::DEBUG);
+				return false;
+			}
+		}
+
+		return true;
+	}
+
+	private static function getDuplicateID($item)
+	{
+		if (empty($item['network']) || in_array($item['network'], Protocol::FEDERATED)) {
+			$condition = ["`uri` = ? AND `uid` = ? AND `network` IN (?, ?, ?, ?)",
+				trim($item['uri']), $item['uid'],
+				Protocol::ACTIVITYPUB, Protocol::DIASPORA, Protocol::DFRN, Protocol::OSTATUS];
+			$existing = self::selectFirst(['id', 'network'], $condition);
+			if (DBA::isResult($existing)) {
+				// We only log the entries with a different user id than 0. Otherwise we would have too many false positives
+				if ($item['uid'] != 0) {
+					Logger::notice('Item already existed for user', [
+						'uri' => $item['uri'],
+						'uid' => $item['uid'],
+						'network' => $item['network'],
+						'existing_id' => $existing["id"],
+						'existing_network' => $existing["network"]
+					]);
+				}
+
+				return $existing["id"];
+			}
+		}
+		return 0;
+	}
+
+	private static function getParentData($item)
+	{
+		// find the parent and snarf the item id and ACLs
+		// and anything else we need to inherit
+
+		$fields = ['uri', 'parent-uri', 'id', 'deleted',
+			'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
+			'wall', 'private', 'forum_mode', 'origin', 'author-id'];
+		$condition = ['uri' => $item['parent-uri'], 'uid' => $item['uid']];
+		$params = ['order' => ['id' => false]];
+		$parent = self::selectFirst($fields, $condition, $params);
+
+		if (!DBA::isResult($parent)) {
+			Logger::info('item parent was not found - ignoring item', ['parent-uri' => $item['parent-uri'], 'uid' => $item['uid']]);
+			return [];
+		} else {
+			// is the new message multi-level threaded?
+			// even though we don't support it now, preserve the info
+			// and re-attach to the conversation parent.
+			if ($parent['uri'] != $parent['parent-uri']) {
+				$item['parent-uri'] = $parent['parent-uri'];
+
+				$condition = ['uri' => $item['parent-uri'],
+					'parent-uri' => $item['parent-uri'],
+					'uid' => $item['uid']];
+				$params = ['order' => ['id' => false]];
+				$toplevel_parent = self::selectFirst($fields, $condition, $params);
+
+				if (DBA::isResult($toplevel_parent)) {
+					$parent = $toplevel_parent;
+				}
+			}
+
+			$item["parent"]        = $parent['id'];
+			$item["deleted"]       = $parent['deleted'];
+			$item["allow_cid"]     = $parent['allow_cid'];
+			$item['allow_gid']     = $parent['allow_gid'];
+			$item['deny_cid']      = $parent['deny_cid'];
+			$item['deny_gid']      = $parent['deny_gid'];
+			$item['parent_origin'] = $parent['origin'];
+
+			// Don't federate received participation messages
+			if ($item['verb'] != Activity::FOLLOW) {
+				$item['wall'] = $parent['wall'];
+			} else {
+				$item['wall'] = false;
+			}
+
+			/*
+			 * If the parent is private, force privacy for the entire conversation
+			 * This differs from the above settings as it subtly allows comments from
+			 * email correspondents to be private even if the overall thread is not.
+			 */
+			if ($parent['private']) {
+				$item['private'] = $parent['private'];
+			}
+
+			/*
+			 * Edge case. We host a public forum that was originally posted to privately.
+			 * The original author commented, but as this is a comment, the permissions
+			 * weren't fixed up so it will still show the comment as private unless we fix it here.
+			 */
+			if ((intval($parent['forum_mode']) == 1) && ($parent['private'] != self::PUBLIC)) {
+				$item['private'] = self::PUBLIC;
+			}
+
+			// If its a post that originated here then tag the thread as "mention"
+			if ($item['origin'] && $item['uid']) {
+				DBA::update('thread', ['mention' => true], ['iid' => $item["parent"]]);
+				Logger::info('tagged thread as mention', ['parent' => $item["parent"], 'uid' => $item['uid']]);
+			}
+
+			// Update the contact relations
+			if ($item['author-id'] != $parent['author-id']) {
+				DBA::update('contact-relation', ['last-interaction' => $item['created']], ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']], true);
+			}
+		}
+
+		return $item;
+	}
+
+	private static function getGravity($item)
+	{
+		$activity = DI::activity();
+
+		if (isset($item['gravity'])) {
+			return intval($item['gravity']);
+		} elseif ($item['parent-uri'] === $item['uri']) {
+			return GRAVITY_PARENT;
+		} elseif ($activity->match($item['verb'], Activity::POST)) {
+			return GRAVITY_COMMENT;
+		} elseif ($activity->match($item['verb'], Activity::FOLLOW)) {
+			return GRAVITY_ACTIVITY;
+		}
+		Logger::info('Unknown gravity for verb', ['verb' => $item['verb']]);
+		return GRAVITY_UNKNOWN;   // Should not happen
+	}
+
+	public static function insert($item, $dummy = false, $notify = false, $dontcache = false)
 	{
 		$orig_item = $item;
 
@@ -1353,6 +1615,8 @@ class Item
 			$item['network'] = trim(($item['network'] ?? '') ?: Protocol::PHANTOM);
 		}
 
+		$uid = intval($item['uid']);
+
 		$item['guid'] = self::guid($item, $notify);
 		$item['uri'] = substr(Strings::escapeTags(trim(($item['uri'] ?? '') ?: self::newURI($item['uid'], $item['guid']))), 0, 255);
 
@@ -1362,100 +1626,24 @@ class Item
 		// Store conversation data
 		$item = Conversation::insert($item);
 
-		/*
-		 * If a Diaspora signature structure was passed in, pull it out of the
-		 * item array and set it aside for later storage.
-		 */
-
-		$dsprsig = null;
-		if (isset($item['dsprsig'])) {
-			$encoded_signature = $item['dsprsig'];
-			$dsprsig = json_decode(base64_decode($item['dsprsig']));
-			unset($item['dsprsig']);
-		}
-
-		$diaspora_signed_text = '';
-		if (isset($item['diaspora_signed_text'])) {
-			$diaspora_signed_text = $item['diaspora_signed_text'];
-			unset($item['diaspora_signed_text']);
-		}
-
-		// Converting the plink
-		/// @TODO Check if this is really still needed
-		if ($item['network'] == Protocol::OSTATUS) {
-			if (isset($item['plink'])) {
-				$item['plink'] = OStatus::convertHref($item['plink']);
-			} elseif (isset($item['uri'])) {
-				$item['plink'] = OStatus::convertHref($item['uri']);
-			}
-		}
-
 		if (!empty($item['thr-parent'])) {
 			$item['parent-uri'] = $item['thr-parent'];
 		}
 
-		$activity = DI::activity();
-
-		if (isset($item['gravity'])) {
-			$item['gravity'] = intval($item['gravity']);
-		} elseif ($item['parent-uri'] === $item['uri']) {
-			$item['gravity'] = GRAVITY_PARENT;
-		} elseif ($activity->match($item['verb'], Activity::POST)) {
-			$item['gravity'] = GRAVITY_COMMENT;
-		} elseif ($activity->match($item['verb'], Activity::FOLLOW)) {
-			$item['gravity'] = GRAVITY_ACTIVITY;
-		} else {
-			$item['gravity'] = GRAVITY_UNKNOWN;   // Should not happen
-			Logger::log('Unknown gravity for verb: ' . $item['verb'], Logger::DEBUG);
-		}
-
-		$uid = intval($item['uid']);
-
-		// check for create date and expire time
-		$expire_interval = DI::config()->get('system', 'dbclean-expire-days', 0);
-
-		$user = DBA::selectFirst('user', ['expire'], ['uid' => $uid]);
-		if (DBA::isResult($user) && ($user['expire'] > 0) && (($user['expire'] < $expire_interval) || ($expire_interval == 0))) {
-			$expire_interval = $user['expire'];
-		}
-
-		if (($expire_interval > 0) && !empty($item['created'])) {
-			$expire_date = time() - ($expire_interval * 86400);
-			$created_date = strtotime($item['created']);
-			if ($created_date < $expire_date) {
-				Logger::notice('Item created before expiration interval.', [
-					'created' => date('c', $created_date),
-					'expired' => date('c', $expire_date),
-					'$item' => $item
-				]);
-				return 0;
-			}
-		}
-
 		/*
 		 * Do we already have this item?
 		 * We have to check several networks since Friendica posts could be repeated
 		 * via OStatus (maybe Diasporsa as well)
 		 */
-		if (empty($item['network']) || in_array($item['network'], Protocol::FEDERATED)) {
-			$condition = ["`uri` = ? AND `uid` = ? AND `network` IN (?, ?, ?, ?)",
-				trim($item['uri']), $item['uid'],
-				Protocol::ACTIVITYPUB, Protocol::DIASPORA, Protocol::DFRN, Protocol::OSTATUS];
-			$existing = self::selectFirst(['id', 'network'], $condition);
-			if (DBA::isResult($existing)) {
-				// We only log the entries with a different user id than 0. Otherwise we would have too many false positives
-				if ($uid != 0) {
-					Logger::notice('Item already existed for user', [
-						'uri' => $item['uri'],
-						'uid' => $uid,
-						'network' => $item['network'],
-						'existing_id' => $existing["id"],
-						'existing_network' => $existing["network"]
-					]);
-				}
+		$duplicate = self::getDuplicateID($item);
+		if ($duplicate) {
+			return $duplicate;
+		}
 
-				return $existing["id"];
-			}
+		// Additional duplicate checks
+		/// @todo Check why the first duplication check returns the item number and the second a 0
+		if (self::isDuplicate($item)) {
+			return 0;
 		}
 
 		$item['wall']          = intval($item['wall'] ?? 0);
@@ -1499,16 +1687,6 @@ class Item
 		$item['inform']        = trim($item['inform'] ?? '');
 		$item['file']          = trim($item['file'] ?? '');
 
-		// When there is no content then we don't post it
-		if ($item['body'].$item['title'] == '') {
-			Logger::notice('No body, no title.');
-			return 0;
-		}
-
-		if (empty($item['vid']) && !empty($item['verb'])) {
-			$item['vid'] = Verb::getID($item['verb']);
-		}
-
 		self::addLanguageToItemArray($item);
 
 		// Items cannot be stored before they happen ...
@@ -1523,60 +1701,26 @@ class Item
 
 		$item['plink'] = ($item['plink'] ?? '') ?: DI::baseUrl() . '/display/' . urlencode($item['guid']);
 
+		$item['gravity'] = self::getGravity($item);
+
 		$default = ['url' => $item['author-link'], 'name' => $item['author-name'],
 			'photo' => $item['author-avatar'], 'network' => $item['network']];
 
 		$item['author-id'] = ($item['author-id'] ?? 0) ?: Contact::getIdForURL($item['author-link'], 0, false, $default);
 
-		if (Contact::isBlocked($item['author-id'])) {
-			Logger::notice('Author is blocked node-wide', ['author-link' => $item['author-link'], 'item-uri' => $item['uri']]);
-			return 0;
-		}
-
-		if (!empty($item['author-link']) && Network::isUrlBlocked($item['author-link'])) {
-			Logger::notice('Author server is blocked', ['author-link' => $item['author-link'], 'item-uri' => $item['uri']]);
-			return 0;
-		}
-
-		if (!empty($uid) && Contact::isBlockedByUser($item['author-id'], $uid)) {
-			Logger::notice('Author is blocked by user', ['author-link' => $item['author-link'], 'uid' => $uid, 'item-uri' => $item['uri']]);
-			return 0;
-		}
+		unset($item['author-link']);
+		unset($item['author-name']);
+		unset($item['author-avatar']);
+		unset($item['author-network']);
 
 		$default = ['url' => $item['owner-link'], 'name' => $item['owner-name'],
 			'photo' => $item['owner-avatar'], 'network' => $item['network']];
 
 		$item['owner-id'] = ($item['owner-id'] ?? 0) ?: Contact::getIdForURL($item['owner-link'], 0, false, $default);
 
-		if (Contact::isBlocked($item['owner-id'])) {
-			Logger::notice('Owner is blocked node-wide', ['owner-link' => $item['owner-link'], 'item-uri' => $item['uri']]);
-			return 0;
-		}
-
-		if (!empty($item['owner-link']) && Network::isUrlBlocked($item['owner-link'])) {
-			Logger::notice('Owner server is blocked', ['owner-link' => $item['owner-link'], 'item-uri' => $item['uri']]);
-			return 0;
-		}
-
-		if (!empty($uid) && Contact::isBlockedByUser($item['owner-id'], $uid)) {
-			Logger::notice('Owner is blocked by user', ['owner-link' => $item['owner-link'], 'uid' => $uid, 'item-uri' => $item['uri']]);
-			return 0;
-		}
-
-		// The causer is set during a thread completion, for example because of a reshare. It countains the responsible actor.
-		if (!empty($uid) && !empty($item['causer-id']) && Contact::isBlockedByUser($item['causer-id'], $uid)) {
-			Logger::notice('Causer is blocked by user', ['causer-link' => $item['causer-link'], 'uid' => $uid, 'item-uri' => $item['uri']]);
-			return 0;
-		}
-
-		if (!empty($uid) && !empty($item['causer-id']) && ($item['parent-uri'] == $item['uri']) && Contact::isIgnoredByUser($item['causer-id'], $uid)) {
-			Logger::notice('Causer is ignored by user', ['causer-link' => $item['causer-link'], 'uid' => $uid, 'item-uri' => $item['uri']]);
-			return 0;
-		}
-
-		// We don't store the causer, we only have it here for the checks above
-		unset($item['causer-id']);
-		unset($item['causer-link']);
+		unset($item['owner-link']);
+		unset($item['owner-name']);
+		unset($item['owner-avatar']);
 
 		// The contact-id should be set before "self::insert" was called - but there seems to be issues sometimes
 		$item["contact-id"] = self::contactId($item);
@@ -1590,178 +1734,34 @@ class Item
 			]);
 		}
 
-		// Checking if there is already an item with the same guid
-		$condition = ['guid' => $item['guid'], 'network' => $item['network'], 'uid' => $item['uid']];
-		if (self::exists($condition)) {
-			Logger::notice('Found already existing item', [
-				'guid' => $item['guid'],
-				'uid' => $item['uid'],
-				'network' => $item['network']
-			]);
+		if (!self::validItem($item)) {
 			return 0;
 		}
 
-		if ($item['verb'] == Activity::FOLLOW) {
-			if (!$item['origin'] && ($item['author-id'] == Contact::getPublicIdByUserId($uid))) {
-				// Our own follow request can be relayed to us. We don't store it to avoid notification chaos.
-				Logger::log("Follow: Don't store not origin follow request from us for " . $item['parent-uri'], Logger::DEBUG);
-				return 0;
-			}
-
-			$condition = ['verb' => Activity::FOLLOW, 'uid' => $item['uid'],
-				'parent-uri' => $item['parent-uri'], 'author-id' => $item['author-id']];
-			if (self::exists($condition)) {
-				// It happens that we receive multiple follow requests by the same author - we only store one.
-				Logger::log('Follow: Found existing follow request from author ' . $item['author-id'] . ' for ' . $item['parent-uri'], Logger::DEBUG);
-				return 0;
-			}
-		}
-
-		// Check for hashtags in the body and repair or add hashtag links
-		self::setHashtags($item);
-
-		// Store tags from the body if this hadn't been handled previously in the protocol classes
-		if (!Tag::existsForPost($item['uri-id'])) {
-			Tag::storeFromBody($item['uri-id'], $item['body']);
-		}
-
+		// We don't store the causer, we only have it here for the checks in the function above
+		unset($item['causer-id']);
+		unset($item['causer-link']);
+		
 		$item['thr-parent'] = $item['parent-uri'];
 
-		$notify_type = Delivery::POST;
-		$allow_cid = '';
-		$allow_gid = '';
-		$deny_cid  = '';
-		$deny_gid  = '';
-
-		if ($item['parent-uri'] === $item['uri']) {
-			$parent_id = 0;
-			$parent_deleted = 0;
-			$allow_cid = $item['allow_cid'];
-			$allow_gid = $item['allow_gid'];
-			$deny_cid  = $item['deny_cid'];
-			$deny_gid  = $item['deny_gid'];
-		} else {
-			// find the parent and snarf the item id and ACLs
-			// and anything else we need to inherit
-
-			$fields = ['uri', 'parent-uri', 'id', 'deleted',
-				'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
-				'wall', 'private', 'forum_mode', 'origin', 'author-id'];
-			$condition = ['uri' => $item['parent-uri'], 'uid' => $item['uid']];
-			$params = ['order' => ['id' => false]];
-			$parent = self::selectFirst($fields, $condition, $params);
-
-			if (DBA::isResult($parent)) {
-				// is the new message multi-level threaded?
-				// even though we don't support it now, preserve the info
-				// and re-attach to the conversation parent.
-
-				if ($parent['uri'] != $parent['parent-uri']) {
-					$item['parent-uri'] = $parent['parent-uri'];
-
-					$condition = ['uri' => $item['parent-uri'],
-						'parent-uri' => $item['parent-uri'],
-						'uid' => $item['uid']];
-					$params = ['order' => ['id' => false]];
-					$toplevel_parent = self::selectFirst($fields, $condition, $params);
-
-					if (DBA::isResult($toplevel_parent)) {
-						$parent = $toplevel_parent;
-					}
-				}
-
-				$parent_id      = $parent['id'];
-				$parent_deleted = $parent['deleted'];
-				$allow_cid      = $parent['allow_cid'];
-				$allow_gid      = $parent['allow_gid'];
-				$deny_cid       = $parent['deny_cid'];
-				$deny_gid       = $parent['deny_gid'];
-
-				// Don't federate received participation messages
-				if ($item['verb'] != Activity::FOLLOW) {
-					$item['wall'] = $parent['wall'];
-				} else {
-					$item['wall'] = false;
-				}
-
-				/*
-				 * If the parent is private, force privacy for the entire conversation
-				 * This differs from the above settings as it subtly allows comments from
-				 * email correspondents to be private even if the overall thread is not.
-				 */
-				if ($parent['private']) {
-					$item['private'] = $parent['private'];
-				}
-
-				/*
-				 * Edge case. We host a public forum that was originally posted to privately.
-				 * The original author commented, but as this is a comment, the permissions
-				 * weren't fixed up so it will still show the comment as private unless we fix it here.
-				 */
-				if ((intval($parent['forum_mode']) == 1) && ($parent['private'] != self::PUBLIC)) {
-					$item['private'] = self::PUBLIC;
-				}
-
-				// If its a post that originated here then tag the thread as "mention"
-				if ($item['origin'] && $item['uid']) {
-					DBA::update('thread', ['mention' => true], ['iid' => $parent_id]);
-					Logger::log('tagged thread ' . $parent_id . ' as mention for user ' . $item['uid'], Logger::DEBUG);
-				}
-
-				// Update the contact relations
-				if ($item['author-id'] != $parent['author-id']) {
-					DBA::update('contact-relation', ['last-interaction' => $item['created']], ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']], true);
-				}
-			} else {
-				/*
-				 * Allow one to see reply tweets from status.net even when
-				 * we don't have or can't see the original post.
-				 */
-				if ($force_parent) {
-					Logger::log('$force_parent=true, reply converted to top-level post.');
-					$parent_id = 0;
-					$item['parent-uri'] = $item['uri'];
-					$item['gravity'] = GRAVITY_PARENT;
-				} else {
-					Logger::log('item parent '.$item['parent-uri'].' for '.$item['uid'].' was not found - ignoring item');
-					return 0;
-				}
-
-				$parent_deleted = 0;
+		if ($item['parent-uri'] != $item['uri']) {
+			$item = self::getParentData($item);
+			if (empty($item)) {
+				return 0;
 			}
-		}
 
-		if (stristr($item['verb'], Activity::POKE)) {
-			$notify_type = Delivery::POKE;
+			$parent_id = $item['parent'];
+			unset($item['parent']);
+			$parent_origin = $item['parent_origin'];
+			unset($item['parent_origin']);
+		} else {
+			$parent_id = 0;
+			$parent_origin = $item['origin'];
 		}
 
 		$item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']);
 		$item['thr-parent-id'] = ItemURI::getIdByURI($item['thr-parent']);
 
-		$condition = ["`uri` = ? AND `network` IN (?, ?) AND `uid` = ?",
-			$item['uri'], $item['network'], Protocol::DFRN, $item['uid']];
-		if (self::exists($condition)) {
-			Logger::log('duplicated item with the same uri found. '.print_r($item,true));
-			return 0;
-		}
-
-		// On Friendica and Diaspora the GUID is unique
-		if (in_array($item['network'], [Protocol::DFRN, Protocol::DIASPORA])) {
-			$condition = ['guid' => $item['guid'], 'uid' => $item['uid']];
-			if (self::exists($condition)) {
-				Logger::log('duplicated item with the same guid found. '.print_r($item,true));
-				return 0;
-			}
-		} elseif ($item['network'] == Protocol::OSTATUS) {
-			// Check for an existing post with the same content. There seems to be a problem with OStatus.
-			$condition = ["`body` = ? AND `network` = ? AND `created` = ? AND `contact-id` = ? AND `uid` = ?",
-					$item['body'], $item['network'], $item['created'], $item['contact-id'], $item['uid']];
-			if (self::exists($condition)) {
-				Logger::log('duplicated item with the same body found. '.print_r($item,true));
-				return 0;
-			}
-		}
-
 		// Is this item available in the global items (with uid=0)?
 		if ($item["uid"] == 0) {
 			$item["global"] = true;
@@ -1773,22 +1773,10 @@ class Item
 		}
 
 		// ACL settings
-		if (strlen($allow_cid) || strlen($allow_gid) || strlen($deny_cid) || strlen($deny_gid)) {
-			$private = self::PRIVATE;
-		} else {
-			$private = $item['private'];
+		if (!empty($item["allow_cid"] . $item["allow_gid"] . $item["deny_cid"] . $item["deny_gid"])) {
+			$item["private"] = self::PRIVATE;
 		}
 
-		$item["allow_cid"] = $allow_cid;
-		$item["allow_gid"] = $allow_gid;
-		$item["deny_cid"] = $deny_cid;
-		$item["deny_gid"] = $deny_gid;
-		$item["private"] = $private;
-		$item["deleted"] = $parent_deleted;
-
-		// Fill the cache field
-		self::putInCache($item);
-
 		if ($notify) {
 			$item['edit'] = false;
 			$item['parent'] = $parent_id;
@@ -1799,34 +1787,13 @@ class Item
 			Hook::callAll('post_remote', $item);
 		}
 
-		// This array field is used to trigger some automatic reactions
-		// It is mainly used in the "post_local" hook.
-		unset($item['api_source']);
-
 		if (!empty($item['cancel'])) {
 			Logger::log('post cancelled by addon.');
 			return 0;
 		}
 
-		/*
-		 * Check for already added items.
-		 * There is a timing issue here that sometimes creates double postings.
-		 * An unique index would help - but the limitations of MySQL (maximum size of index values) prevent this.
-		 */
-		if ($item["uid"] == 0) {
-			if (self::exists(['uri' => trim($item['uri']), 'uid' => 0])) {
-				Logger::log('Global item already stored. URI: '.$item['uri'].' on network '.$item['network'], Logger::DEBUG);
-				return 0;
-			}
-		}
-
-		Logger::log('' . print_r($item,true), Logger::DATA);
-
-		if (array_key_exists('file', $item)) {
-			$files = $item['file'];
-			unset($item['file']);
-		} else {
-			$files = '';
+		if (empty($item['vid']) && !empty($item['verb'])) {
+			$item['vid'] = Verb::getID($item['verb']);
 		}
 
 		// Creates or assigns the permission set
@@ -1838,31 +1805,73 @@ class Item
 			$item['deny_gid']
 		);
 
-		$item['allow_cid'] = null;
-		$item['allow_gid'] = null;
-		$item['deny_cid'] = null;
-		$item['deny_gid'] = null;
+		unset($item['allow_cid']);
+		unset($item['allow_gid']);
+		unset($item['deny_cid']);
+		unset($item['deny_gid']);
+
+		// This array field is used to trigger some automatic reactions
+		// It is mainly used in the "post_local" hook.
+		unset($item['api_source']);
+
+		// Filling item related side tables
+		if (!empty($item['dsprsig'])) {
+			$dsprsig = json_decode(base64_decode($item['dsprsig']));
+
+			/*
+			 * Friendica servers lower than 3.4.3-2 had double encoded the signature ...
+			 * We can check for this condition when we decode and encode the stuff again.
+			 */
+			if (base64_encode(base64_decode(base64_decode($dsprsig->signature))) == base64_decode($dsprsig->signature)) {
+				$dsprsig->signature = base64_decode($dsprsig->signature);
+				Logger::log("Repaired double encoded signature from handle ".$dsprsig->signer, Logger::DEBUG);
+			}
+
+			if (!empty($dsprsig->signed_text) && empty($dsprsig->signature) && empty($dsprsig->signer)) {
+				DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $dsprsig->signed_text], true);
+			}
+		}
+
+		unset($item['dsprsig']);
+
+		if (!empty($item['diaspora_signed_text'])) {
+			DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $item['diaspora_signed_text']], true);
+		}
+
+		unset($item['diaspora_signed_text']);
+
+		if (array_key_exists('file', $item) && !empty($item['file'])) {
+			Category::storeTextByURIId($item['uri-id'], $item['uid'], $item['file']);
+		}
+
+		unset($item['file']);
+
+		$delivery_data = Post\DeliveryData::extractFields($item);
+		unset($item['postopts']);
+		unset($item['inform']);
+
+		// Check for hashtags in the body and repair or add hashtag links
+		self::setHashtags($item);
+
+		// Store tags from the body if this hadn't been handled previously in the protocol classes
+		if (!Tag::existsForPost($item['uri-id'])) {
+			Tag::storeFromBody($item['uri-id'], $item['body']);
+		}
+		
+		// Fill the cache field
+		self::putInCache($item);
+
+		if (stristr($item['verb'], Activity::POKE)) {
+			$notify_type = Delivery::POKE;
+		} else {
+			$notify_type = Delivery::POST;
+		}
 
 		// We are doing this outside of the transaction to avoid timing problems
 		if (!self::insertActivity($item)) {
 			self::insertContent($item);
 		}
 
-		$delivery_data = Post\DeliveryData::extractFields($item);
-
-		unset($item['postopts']);
-		unset($item['inform']);
-
-		// These fields aren't stored anymore in the item table, they are fetched upon request
-		unset($item['author-link']);
-		unset($item['author-name']);
-		unset($item['author-avatar']);
-		unset($item['author-network']);
-
-		unset($item['owner-link']);
-		unset($item['owner-name']);
-		unset($item['owner-avatar']);
-
 		$like_no_comment = DI::config()->get('system', 'like_no_comment');
 
 		DBA::transaction();
@@ -1926,25 +1935,6 @@ class Item
 			DBA::update('item', ['changed' => DateTimeFormat::utcNow()], ['id' => $parent_id]);
 		}
 
-		if ($dsprsig) {
-			/*
-			 * Friendica servers lower than 3.4.3-2 had double encoded the signature ...
-			 * We can check for this condition when we decode and encode the stuff again.
-			 */
-			if (base64_encode(base64_decode(base64_decode($dsprsig->signature))) == base64_decode($dsprsig->signature)) {
-				$dsprsig->signature = base64_decode($dsprsig->signature);
-				Logger::log("Repaired double encoded signature from handle ".$dsprsig->signer, Logger::DEBUG);
-			}
-
-			if (!empty($dsprsig->signed_text) && empty($dsprsig->signature) && empty($dsprsig->signer)) {
-				DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $dsprsig->signed_text], true);
-			}
-		}
-
-		if (!empty($diaspora_signed_text)) {
-			DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $diaspora_signed_text], true);
-		}
-
 		if ($item['parent-uri'] === $item['uri']) {
 			self::addThread($current_post);
 		} else {
@@ -1957,14 +1947,6 @@ class Item
 
 		DBA::commit();
 
-		/*
-		 * Due to deadlock issues with the "term" table we are doing these steps after the commit.
-		 * This is not perfect - but a workable solution until we found the reason for the problem.
-		 */
-		if (!empty($files)) {
-			Category::storeTextByURIId($item['uri-id'], $item['uid'], $files);
-		}
-
 		// In that function we check if this is a forum post. Additionally we delete the item under certain circumstances
 		if (self::tagDeliver($item['uid'], $current_post)) {
 			// Get the user information for the logging
@@ -1999,7 +1981,7 @@ class Item
 
 		check_user_notification($current_post);
 
-		$transmit = $notify || ($item['visible'] && ((!empty($parent) && $parent['origin']) || $item['origin']));
+		$transmit = $notify || ($item['visible'] && ($parent_origin || $item['origin']));
 
 		if ($transmit) {
 			$transmit_item = Item::selectFirst(['verb', 'origin'], ['id' => $item['id']]);
diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php
index 07465c522..c7882d035 100644
--- a/src/Protocol/OStatus.php
+++ b/src/Protocol/OStatus.php
@@ -1001,7 +1001,7 @@ class OStatus
 
 		// Even more worse workaround for GNU Social ;-)
 		if ($xml == '') {
-			$related_guess = OStatus::convertHref($related_uri);
+			$related_guess = self::convertHref($related_uri);
 			$curlResult = Network::curl(str_replace('/notice/', '/api/statuses/show/', $related_guess).'.atom');
 
 			if ($curlResult->isSuccess()) {
@@ -1175,7 +1175,7 @@ class OStatus
 	 *
 	 * @return string URL in the format http(s)://....
 	 */
-	public static function convertHref($href)
+	private static function convertHref($href)
 	{
 		$elements = explode(":", $href);
 

From c4062ddb3b54905bddb40f78d26b40210f466eb4 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 12 May 2020 21:49:12 +0000
Subject: [PATCH 2/6] Removed "insert" parameter

---
 src/Model/Item.php    | 12 +++++-------
 src/Protocol/DFRN.php |  2 +-
 src/Protocol/Feed.php |  2 +-
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 78dd6aff2..e2ea9eb81 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -36,13 +36,11 @@ use Friendica\Model\Post\Category;
 use Friendica\Protocol\Activity;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Protocol\Diaspora;
-use Friendica\Protocol\OStatus;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Map;
 use Friendica\Util\Network;
 use Friendica\Util\Security;
 use Friendica\Util\Strings;
-use Friendica\Util\XML;
 use Friendica\Worker\Delivery;
 use Text_LanguageDetect;
 use Friendica\Repository\PermissionSet as RepPermissionSet;
@@ -1595,7 +1593,7 @@ class Item
 		return GRAVITY_UNKNOWN;   // Should not happen
 	}
 
-	public static function insert($item, $dummy = false, $notify = false, $dontcache = false)
+	public static function insert($item, $notify = false, $dontcache = false)
 	{
 		$orig_item = $item;
 
@@ -2277,7 +2275,7 @@ class Item
 			}
 		}
 
-		$distributed = self::insert($item, false, $notify, true);
+		$distributed = self::insert($item, $notify, true);
 
 		if (!$distributed) {
 			Logger::log("Distributed public item " . $itemid . " for user " . $uid . " wasn't stored", Logger::DEBUG);
@@ -2344,7 +2342,7 @@ class Item
 				$item['contact-id'] = $item['author-id'];
 			}
 
-			$public_shadow = self::insert($item, false, false, true);
+			$public_shadow = self::insert($item, false, true);
 
 			Logger::log("Stored public shadow for thread ".$itemid." under id ".$public_shadow, Logger::DEBUG);
 		}
@@ -2402,7 +2400,7 @@ class Item
 		unset($item['inform']);
 		$item['contact-id'] = Contact::getIdForURL($item['author-link']);
 
-		$public_shadow = self::insert($item, false, false, true);
+		$public_shadow = self::insert($item, false, true);
 
 		Logger::log("Stored public shadow for comment ".$item['uri']." under id ".$public_shadow, Logger::DEBUG);
 
@@ -2788,7 +2786,7 @@ class Item
 
 		if ($contact['network'] != Protocol::FEED) {
 			// Store the original post
-			$result = self::insert($datarray2, false, false);
+			$result = self::insert($datarray2);
 			Logger::log('remote-self post original item - Contact '.$contact['url'].' return '.$result.' Item '.print_r($datarray2, true), Logger::DEBUG);
 		} else {
 			$datarray["app"] = "Feed";
diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php
index fbcaf8d4b..08ce16271 100644
--- a/src/Protocol/DFRN.php
+++ b/src/Protocol/DFRN.php
@@ -2584,7 +2584,7 @@ class DFRN
 			// Turn this into a wall post.
 			$notify = Item::isRemoteSelf($importer, $item);
 
-			$posted_id = Item::insert($item, false, $notify);
+			$posted_id = Item::insert($item, $notify);
 
 			if ($notify) {
 				$posted_id = $notify;
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index baf439dc0..35e2cd577 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -511,7 +511,7 @@ class Feed {
 					$notify = PRIORITY_MEDIUM;
 				}
 
-				$id = Item::insert($item, false, $notify);
+				$id = Item::insert($item, $notify);
 
 				Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id);
 

From 13bd43f3f6c7458e84c65b342632d116a1bf9049 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 13 May 2020 05:48:26 +0000
Subject: [PATCH 3/6] Store the diaspora signature differently

---
 src/Model/Item.php    | 47 ++++++++++++++++---------------------------
 src/Protocol/DFRN.php |  6 +++++-
 2 files changed, 22 insertions(+), 31 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index e2ea9eb81..bac08e63a 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1454,7 +1454,7 @@ class Item
 		if ($item['verb'] == Activity::FOLLOW) {
 			if (!$item['origin'] && ($item['author-id'] == Contact::getPublicIdByUserId($item['uid']))) {
 				// Our own follow request can be relayed to us. We don't store it to avoid notification chaos.
-				Logger::log("Follow: Don't store not origin follow request from us for " . $item['parent-uri'], Logger::DEBUG);
+				Logger::info("Follow: Don't store not origin follow request", ['parent-uri' => $item['parent-uri']]);
 				return false;
 			}
 
@@ -1462,7 +1462,7 @@ class Item
 				'parent-uri' => $item['parent-uri'], 'author-id' => $item['author-id']];
 			if (self::exists($condition)) {
 				// It happens that we receive multiple follow requests by the same author - we only store one.
-				Logger::log('Follow: Found existing follow request from author ' . $item['author-id'] . ' for ' . $item['parent-uri'], Logger::DEBUG);
+				Logger::info('Follow: Found existing follow request from author', ['author-id' => $item['author-id'], 'parent-uri' => $item['parent-uri']]);
 				return false;
 			}
 		}
@@ -1706,20 +1706,11 @@ class Item
 
 		$item['author-id'] = ($item['author-id'] ?? 0) ?: Contact::getIdForURL($item['author-link'], 0, false, $default);
 
-		unset($item['author-link']);
-		unset($item['author-name']);
-		unset($item['author-avatar']);
-		unset($item['author-network']);
-
 		$default = ['url' => $item['owner-link'], 'name' => $item['owner-name'],
 			'photo' => $item['owner-avatar'], 'network' => $item['network']];
 
 		$item['owner-id'] = ($item['owner-id'] ?? 0) ?: Contact::getIdForURL($item['owner-link'], 0, false, $default);
 
-		unset($item['owner-link']);
-		unset($item['owner-name']);
-		unset($item['owner-avatar']);
-
 		// The contact-id should be set before "self::insert" was called - but there seems to be issues sometimes
 		$item["contact-id"] = self::contactId($item);
 
@@ -1739,7 +1730,18 @@ class Item
 		// We don't store the causer, we only have it here for the checks in the function above
 		unset($item['causer-id']);
 		unset($item['causer-link']);
-		
+
+		// We don't store these fields anymore in the item table
+		unset($item['author-link']);
+		unset($item['author-name']);
+		unset($item['author-avatar']);
+		unset($item['author-network']);
+
+		unset($item['owner-link']);
+		unset($item['owner-name']);
+		unset($item['owner-avatar']);
+
+
 		$item['thr-parent'] = $item['parent-uri'];
 
 		if ($item['parent-uri'] != $item['uri']) {
@@ -1813,37 +1815,22 @@ class Item
 		unset($item['api_source']);
 
 		// Filling item related side tables
-		if (!empty($item['dsprsig'])) {
-			$dsprsig = json_decode(base64_decode($item['dsprsig']));
-
-			/*
-			 * Friendica servers lower than 3.4.3-2 had double encoded the signature ...
-			 * We can check for this condition when we decode and encode the stuff again.
-			 */
-			if (base64_encode(base64_decode(base64_decode($dsprsig->signature))) == base64_decode($dsprsig->signature)) {
-				$dsprsig->signature = base64_decode($dsprsig->signature);
-				Logger::log("Repaired double encoded signature from handle ".$dsprsig->signer, Logger::DEBUG);
-			}
-
-			if (!empty($dsprsig->signed_text) && empty($dsprsig->signature) && empty($dsprsig->signer)) {
-				DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $dsprsig->signed_text], true);
-			}
-		}
-
-		unset($item['dsprsig']);
 
+		// Diaspora signature
 		if (!empty($item['diaspora_signed_text'])) {
 			DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $item['diaspora_signed_text']], true);
 		}
 
 		unset($item['diaspora_signed_text']);
 
+		// Attached file links
 		if (array_key_exists('file', $item) && !empty($item['file'])) {
 			Category::storeTextByURIId($item['uri-id'], $item['uid'], $item['file']);
 		}
 
 		unset($item['file']);
 
+		// Delivery relevant data
 		$delivery_data = Post\DeliveryData::extractFields($item);
 		unset($item['postopts']);
 		unset($item['inform']);
diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php
index 08ce16271..7d8bc9dc5 100644
--- a/src/Protocol/DFRN.php
+++ b/src/Protocol/DFRN.php
@@ -2384,7 +2384,11 @@ class DFRN
 		// We store the data from "dfrn:diaspora_signature" in a different table, this is done in "Item::insert"
 		$dsprsig = XML::unescape(XML::getFirstNodeValue($xpath, "dfrn:diaspora_signature/text()", $entry));
 		if ($dsprsig != "") {
-			$item["dsprsig"] = $dsprsig;
+			$signature = json_decode(base64_decode($dsprsig));
+			// We don't store the old style signatures anymore that also contained the "signature" and "signer"
+			if (!empty($signature->signed_text) && empty($signature->signature) && empty($signature->signer)) {
+				$item["diaspora_signed_text"] = $signature->signed_text;
+			}
 		}
 
 		$item["verb"] = XML::getFirstNodeValue($xpath, "activity:verb/text()", $entry);

From 7ea40ea7964f7607b5becf5315c93aca1c03c61c Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 13 May 2020 18:45:31 +0000
Subject: [PATCH 4/6] Rearranged calls

---
 src/Model/Item.php | 89 +++++++++++++++++++++-------------------------
 1 file changed, 41 insertions(+), 48 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index bac08e63a..d3b2054de 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1685,8 +1685,6 @@ class Item
 		$item['inform']        = trim($item['inform'] ?? '');
 		$item['file']          = trim($item['file'] ?? '');
 
-		self::addLanguageToItemArray($item);
-
 		// Items cannot be stored before they happen ...
 		if ($item['created'] > DateTimeFormat::utcNow()) {
 			$item['created'] = DateTimeFormat::utcNow();
@@ -1699,30 +1697,21 @@ class Item
 
 		$item['plink'] = ($item['plink'] ?? '') ?: DI::baseUrl() . '/display/' . urlencode($item['guid']);
 
+		$item['language'] = self::getLanguage($item);
+
 		$item['gravity'] = self::getGravity($item);
 
 		$default = ['url' => $item['author-link'], 'name' => $item['author-name'],
 			'photo' => $item['author-avatar'], 'network' => $item['network']];
-
 		$item['author-id'] = ($item['author-id'] ?? 0) ?: Contact::getIdForURL($item['author-link'], 0, false, $default);
 
 		$default = ['url' => $item['owner-link'], 'name' => $item['owner-name'],
 			'photo' => $item['owner-avatar'], 'network' => $item['network']];
-
 		$item['owner-id'] = ($item['owner-id'] ?? 0) ?: Contact::getIdForURL($item['owner-link'], 0, false, $default);
 
 		// The contact-id should be set before "self::insert" was called - but there seems to be issues sometimes
 		$item["contact-id"] = self::contactId($item);
 
-		if ($item['network'] == Protocol::PHANTOM) {
-			$item['network'] = Protocol::DFRN;
-			Logger::notice('Missing network, setting to {network}.', [
-				'uri' => $item["uri"],
-				'network' => $item['network'],
-				'callstack' => System::callstack()
-			]);
-		}
-
 		if (!self::validItem($item)) {
 			return 0;
 		}
@@ -1741,7 +1730,6 @@ class Item
 		unset($item['owner-name']);
 		unset($item['owner-avatar']);
 
-
 		$item['thr-parent'] = $item['parent-uri'];
 
 		if ($item['parent-uri'] != $item['uri']) {
@@ -1814,34 +1802,9 @@ class Item
 		// It is mainly used in the "post_local" hook.
 		unset($item['api_source']);
 
-		// Filling item related side tables
-
-		// Diaspora signature
-		if (!empty($item['diaspora_signed_text'])) {
-			DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $item['diaspora_signed_text']], true);
-		}
-
-		unset($item['diaspora_signed_text']);
-
-		// Attached file links
-		if (array_key_exists('file', $item) && !empty($item['file'])) {
-			Category::storeTextByURIId($item['uri-id'], $item['uid'], $item['file']);
-		}
-
-		unset($item['file']);
-
-		// Delivery relevant data
-		$delivery_data = Post\DeliveryData::extractFields($item);
-		unset($item['postopts']);
-		unset($item['inform']);
 
 		// Check for hashtags in the body and repair or add hashtag links
 		self::setHashtags($item);
-
-		// Store tags from the body if this hadn't been handled previously in the protocol classes
-		if (!Tag::existsForPost($item['uri-id'])) {
-			Tag::storeFromBody($item['uri-id'], $item['body']);
-		}
 		
 		// Fill the cache field
 		self::putInCache($item);
@@ -1852,6 +1815,8 @@ class Item
 			$notify_type = Delivery::POST;
 		}
 
+		$body = $item['body'];
+		
 		// We are doing this outside of the transaction to avoid timing problems
 		if (!self::insertActivity($item)) {
 			self::insertContent($item);
@@ -1860,6 +1825,37 @@ class Item
 		$like_no_comment = DI::config()->get('system', 'like_no_comment');
 
 		DBA::transaction();
+
+		// Filling item related side tables
+
+		// Diaspora signature
+		if (!empty($item['diaspora_signed_text'])) {
+			DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $item['diaspora_signed_text']], true);
+		}
+
+		unset($item['diaspora_signed_text']);
+
+		// Attached file links
+		if (!empty($item['file'])) {
+			Category::storeTextByURIId($item['uri-id'], $item['uid'], $item['file']);
+		}
+
+		unset($item['file']);
+
+		// Delivery relevant data
+		$delivery_data = Post\DeliveryData::extractFields($item);
+		unset($item['postopts']);
+		unset($item['inform']);
+
+		if (!empty($item['origin']) || !empty($item['wall']) || !empty($delivery_data['postopts']) || !empty($delivery_data['inform'])) {
+			Post\DeliveryData::insert($item['uri-id'], $delivery_data);
+		}
+
+		// Store tags from the body if this hadn't been handled previously in the protocol classes
+		if (!Tag::existsForPost($item['uri-id'])) {
+			Tag::storeFromBody($item['uri-id'], $body);
+		}
+		
 		$ret = DBA::insert('item', $item);
 
 		// When the item was successfully stored we fetch the ID of the item.
@@ -1925,11 +1921,6 @@ class Item
 		} else {
 			self::updateThread($parent_id);
 		}
-
-		if (!empty($item['origin']) || !empty($item['wall']) || !empty($delivery_data['postopts']) || !empty($delivery_data['inform'])) {
-			Post\DeliveryData::insert($item['uri-id'], $delivery_data);
-		}
-
 		DBA::commit();
 
 		// In that function we check if this is a forum post. Additionally we delete the item under certain circumstances
@@ -2402,20 +2393,22 @@ class Item
 	 * Adds a language specification in a "language" element of given $arr.
 	 * Expects "body" element to exist in $arr.
 	 *
-	 * @param $item
+	 * @param array $item
+	 * @return string detected language
 	 * @throws \Text_LanguageDetect_Exception
 	 */
-	private static function addLanguageToItemArray(&$item)
+	private static function getLanguage(array $item)
 	{
 		$naked_body = BBCode::toPlaintext($item['body'], false);
 
 		$ld = new Text_LanguageDetect();
 		$ld->setNameMode(2);
 		$languages = $ld->detect($naked_body, 3);
-
 		if (is_array($languages)) {
-			$item['language'] = json_encode($languages);
+			return json_encode($languages);
 		}
+
+		return '';
 	}
 
 	/**

From 04005f287834507527411d9647bf06b519b5a768 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 13 May 2020 19:26:59 +0000
Subject: [PATCH 5/6] Added documentation

---
 src/Model/Item.php | 40 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index d3b2054de..58654e877 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1331,7 +1331,13 @@ class Item
 		}
 	}
 
-	private static function isDuplicate($item)
+	/**
+	 * Check if the item array is a duplicate
+	 *
+	 * @param array $item
+	 * @return boolean is it a duplicate?
+	 */
+	private static function isDuplicate(array $item)
 	{
 		// Checking if there is already an item with the same guid
 		$condition = ['guid' => $item['guid'], 'network' => $item['network'], 'uid' => $item['uid']];
@@ -1381,7 +1387,13 @@ class Item
 		return false;
 	}
 
-	private static function validItem($item)
+	/**
+	 * Check if the item array is valid
+	 *
+	 * @param array $item
+	 * @return boolean item is valid
+	 */
+	private static function validItem(array $item)
 	{
 		// When there is no content then we don't post it
 		if ($item['body'].$item['title'] == '') {
@@ -1470,7 +1482,13 @@ class Item
 		return true;
 	}
 
-	private static function getDuplicateID($item)
+	/**
+	 * Return the id of the givven item array when it had been stored before
+	 *
+	 * @param array $item
+	 * @return integer item id
+	 */
+	private static function getDuplicateID(array $item)
 	{
 		if (empty($item['network']) || in_array($item['network'], Protocol::FEDERATED)) {
 			$condition = ["`uri` = ? AND `uid` = ? AND `network` IN (?, ?, ?, ?)",
@@ -1495,7 +1513,13 @@ class Item
 		return 0;
 	}
 
-	private static function getParentData($item)
+	/**
+	 * Fetch parent data for the given item array
+	 *
+	 * @param array $item
+	 * @return array item array with parent data
+	 */
+	private static function getParentData(array $item)
 	{
 		// find the parent and snarf the item id and ACLs
 		// and anything else we need to inherit
@@ -1576,7 +1600,13 @@ class Item
 		return $item;
 	}
 
-	private static function getGravity($item)
+	/**
+	 * Get the gravity for the given item array
+	 *
+	 * @param array $item
+	 * @return integer gravity
+	 */
+	private static function getGravity(array $item)
 	{
 		$activity = DI::activity();
 

From 3dc231ffd6f447986572a841866922d6eed902cc Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Thu, 14 May 2020 03:48:26 +0000
Subject: [PATCH 6/6] Code standards

---
 src/Model/Item.php | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 58654e877..f10537d75 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1379,7 +1379,7 @@ class Item
 		 * There is a timing issue here that sometimes creates double postings.
 		 * An unique index would help - but the limitations of MySQL (maximum size of index values) prevent this.
 		 */
-		if (($item["uid"] == 0) && self::exists(['uri' => trim($item['uri']), 'uid' => 0])) {
+		if (($item['uid'] == 0) && self::exists(['uri' => trim($item['uri']), 'uid' => 0])) {
 			Logger::notice('Global item already stored.', ['uri' => $item['uri'], 'network' => $item['network']]);
 			return true;
 		}
@@ -1393,7 +1393,7 @@ class Item
 	 * @param array $item
 	 * @return boolean item is valid
 	 */
-	private static function validItem(array $item)
+	private static function isValid(array $item)
 	{
 		// When there is no content then we don't post it
 		if ($item['body'].$item['title'] == '') {
@@ -1483,7 +1483,7 @@ class Item
 	}
 
 	/**
-	 * Return the id of the givven item array when it had been stored before
+	 * Return the id of the given item array if it has been stored before
 	 *
 	 * @param array $item
 	 * @return integer item id
@@ -1742,7 +1742,7 @@ class Item
 		// The contact-id should be set before "self::insert" was called - but there seems to be issues sometimes
 		$item["contact-id"] = self::contactId($item);
 
-		if (!self::validItem($item)) {
+		if (!self::isValid($item)) {
 			return 0;
 		}