From d0c36d0e8913cf5a7eb2b92b8564c06606938e58 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 24 Jun 2018 14:33:07 +0000
Subject: [PATCH 01/11] New table for storing item content

---
 boot.php                     |  2 +-
 src/Database/DBStructure.php | 28 +++++++++++++++++++++++++++-
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/boot.php b/boot.php
index 9bc02584c2..ac21d108a3 100644
--- a/boot.php
+++ b/boot.php
@@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'The Tazmans Flax-lily');
 define('FRIENDICA_VERSION',      '2018.08-dev');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
-define('DB_UPDATE_VERSION',      1270);
+define('DB_UPDATE_VERSION',      1271);
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
 /**
diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php
index a089ab8942..19930b5e4c 100644
--- a/src/Database/DBStructure.php
+++ b/src/Database/DBStructure.php
@@ -1154,7 +1154,7 @@ class DBStructure
 						]
 				];
 		$database["item"] = [
-				"comment" => "All posts",
+				"comment" => "Structure for all posts",
 				"fields" => [
 						"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]],
 						"guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "A unique identifier for this item"],
@@ -1251,6 +1251,32 @@ class DBStructure
 						"uid_ownerlink" => ["uid","owner-link(190)"],
 						]
 				];
+		$database["item-content"] = [
+				"comment" => "Content for all posts",
+				"fields" => [
+						"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]],
+						"uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
+						"uri-plink-hash" => ["type" => "char(80)", "not null" => "1", "default" => "", "comment" => "SHA-1 hash from uri and plink"],
+						"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
+						"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
+						"body" => ["type" => "mediumtext", "comment" => "item body content"],
+						"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
+						"coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
+						"app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"],
+						"rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
+						"rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"],
+						"object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"],
+						"object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"],
+						"target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"],
+						"target" => ["type" => "text", "comment" => "JSON encoded target structure if used"],
+						"plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"],
+						],
+				"indexes" => [
+						"PRIMARY" => ["id"],
+						"uri-plink-hash" => ["UNIQUE", "uri-plink-hash"],
+						"uri" => ["uri(191)"],
+						]
+				];
 		$database["locks"] = [
 				"comment" => "",
 				"fields" => [

From 58d61810ea8c487e1bc7b97c7d5e4146a152bcf2 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 24 Jun 2018 21:41:49 +0000
Subject: [PATCH 02/11] We now store item content centrally

---
 src/Model/Item.php | 75 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 72 insertions(+), 3 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index d6bfef39fb..18d89d55ae 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -56,6 +56,10 @@ class Item extends BaseObject
 			'author-id', 'author-link', 'owner-link', 'contact-uid',
 			'signed_text', 'signature', 'signer'];
 
+	const CONTENT_FIELDLIST = ['title', 'content-warning', 'body', 'location',
+			'coord', 'app', 'rendered-hash', 'rendered-html',
+			'object-type', 'object', 'target-type', 'target'];
+
 	/**
 	 * @brief Fetch a single item row
 	 *
@@ -66,6 +70,14 @@ class Item extends BaseObject
 	{
 		$row = dba::fetch($stmt);
 
+		// Fetch data from the item-content table whenever there is content there
+		foreach (self::CONTENT_FIELDLIST as $field) {
+			if (isset($row[$field]) && !empty($row['item-content-' . $field])) {
+				$row[$field] = $row['item-content-' . $field];
+				unset($row['item-content-' . $field]);
+			}
+		}
+
 		// We prefer the data from the user's contact over the public one
 		if (!empty($row['author-link']) && !empty($row['contact-link']) &&
 			($row['author-link'] == $row['contact-link'])) {
@@ -449,6 +461,10 @@ class Item extends BaseObject
 			$joins .= " LEFT JOIN `sign` ON `sign`.`iid` = `item`.`id`";
 		}
 
+		if (strpos($sql_commands, "`item-content`.") !== false) {
+			$joins .= " LEFT JOIN `item-content` ON `item-content`.`uri` = `item`.`uri`";
+		}
+
 		if ((strpos($sql_commands, "`parent-item`.") !== false) || (strpos($sql_commands, "`parent-author`.") !== false)) {
 			$joins .= " STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`id` = `item`.`parent`";
 		}
@@ -474,10 +490,13 @@ class Item extends BaseObject
 		foreach ($fields as $table => $table_fields) {
 			foreach ($table_fields as $field => $select) {
 				if (empty($selected) || in_array($select, $selected)) {
+					if (in_array($select, self::CONTENT_FIELDLIST)) {
+						$selection[] = "`item-content`.`".$select."` AS `item-content-" . $select . "`";
+					}
 					if (is_int($field)) {
-						$selection[] = "`" . $table . "`.`".$select."`";
+						$selection[] = "`" . $table . "`.`" . $select . "`";
 					} else {
-						$selection[] = "`" . $table . "`.`" . $field . "` AS `".$select ."`";
+						$selection[] = "`" . $table . "`.`" . $field . "` AS `" . $select . "`";
 					}
 				}
 			}
@@ -536,7 +555,7 @@ class Item extends BaseObject
 		// We cannot simply expand the condition to check for origin entries
 		// The condition needn't to be a simple array but could be a complex condition.
 		// And we have to execute this query before the update to ensure to fetch the same data.
-		$items = dba::select('item', ['id', 'origin'], $condition);
+		$items = dba::select('item', ['id', 'origin', 'uri'], $condition);
 
 		$success = dba::update('item', $fields, $condition);
 
@@ -549,6 +568,7 @@ class Item extends BaseObject
 		$rows = dba::affected_rows();
 
 		while ($item = dba::fetch($items)) {
+			self::updateContent($fields, ['uri' => $item['uri']]);
 			Term::insertFromTagFieldByItemId($item['id']);
 			Term::insertFromFileFieldByItemId($item['id']);
 			self::updateThread($item['id']);
@@ -1260,6 +1280,7 @@ class Item extends BaseObject
 		logger('' . print_r($item,true), LOGGER_DATA);
 
 		dba::transaction();
+		self::insertContent($item);
 		$ret = dba::insert('item', $item);
 
 		// When the item was successfully stored we fetch the ID of the item.
@@ -1407,6 +1428,54 @@ class Item extends BaseObject
 		return $current_post;
 	}
 
+	/**
+	 * @brief Insert a new item content entry
+	 *
+	 * @param array $item The item fields that are to be inserted
+	 */
+	private static function insertContent(&$item)
+	{
+		logger('Insert content for URI '.$item['uri']);
+
+		$fields = ['uri' => $item['uri'], 'title' => $item['title'],
+			'content-warning' => $item['content-warning'],
+			'body' => $item['body'], 'location' => $item['location'],
+			'coord' => $item['coord'], 'app' => $item['app'],
+			'rendered-hash' => $item['rendered-hash'],
+			'rendered-html' => $item['rendered-html'],
+			'object-type' => $item['object-type'],
+			'object' => $item['object'], 'target-type' => $item['target-type'],
+			'target' => $item['target'], 'plink' => $item['plink'],
+			'uri-plink-hash' => hash('sha1', $item['plink']).hash('sha1', $item['uri'])];
+
+		dba::insert('item-content', $fields, true);
+	}
+
+	/**
+	 * @brief Update existing item content entries
+	 *
+	 * @param array $item The item fields that are to be changed
+	 * @param array $condition The condition for finding the item content entries
+	 */
+	private static function updateContent($item, $condition)
+	{
+		// We have to select only the fields from the "item-content" table
+		$fields = [];
+		foreach (self::CONTENT_FIELDLIST as $field) {
+			if (isset($item[$field])) {
+				$fields[$field] = $item[$field];
+			}
+		}
+
+		if (empty($fields)) {
+			return;
+		}
+
+		logger('Update content for URI '.$condition['uri']);
+
+		dba::update('item-content', $fields, $condition, true);
+	}
+
 	/**
 	 * @brief Distributes public items to the receivers
 	 *

From 941bab1096dc3a383ef042e1ec5b403fc660c630 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 24 Jun 2018 23:09:13 +0000
Subject: [PATCH 03/11] We now only store the content in item-content

---
 src/Model/Item.php | 44 +++++++++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 18d89d55ae..3936a3af47 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -72,7 +72,7 @@ class Item extends BaseObject
 
 		// Fetch data from the item-content table whenever there is content there
 		foreach (self::CONTENT_FIELDLIST as $field) {
-			if (isset($row[$field]) && !empty($row['item-content-' . $field])) {
+			if (!empty($row['item-content-' . $field])) {
 				$row[$field] = $row['item-content-' . $field];
 				unset($row['item-content-' . $field]);
 			}
@@ -557,18 +557,29 @@ class Item extends BaseObject
 		// And we have to execute this query before the update to ensure to fetch the same data.
 		$items = dba::select('item', ['id', 'origin', 'uri'], $condition);
 
-		$success = dba::update('item', $fields, $condition);
-
-		if (!$success) {
-			dba::close($items);
-			dba::rollback();
-			return false;
+		$content_fields = [];
+		foreach (self::CONTENT_FIELDLIST as $field) {
+			if (isset($fields[$field])) {
+				$content_fields[$field] = $fields[$field];
+				unset($fields[$field]);
+			}
 		}
 
+		if (!empty($fields)) {
+			$success = dba::update('item', $fields, $condition);
+
+			if (!$success) {
+				dba::close($items);
+				dba::rollback();
+				return false;
+			}
+		}
+
+		// When there is no content for the "old" item table, this will count the fetched items
 		$rows = dba::affected_rows();
 
 		while ($item = dba::fetch($items)) {
-			self::updateContent($fields, ['uri' => $item['uri']]);
+			self::updateContent($content_fields, ['uri' => $item['uri']]);
 			Term::insertFromTagFieldByItemId($item['id']);
 			Term::insertFromFileFieldByItemId($item['id']);
 			self::updateThread($item['id']);
@@ -1437,17 +1448,16 @@ class Item extends BaseObject
 	{
 		logger('Insert content for URI '.$item['uri']);
 
-		$fields = ['uri' => $item['uri'], 'title' => $item['title'],
-			'content-warning' => $item['content-warning'],
-			'body' => $item['body'], 'location' => $item['location'],
-			'coord' => $item['coord'], 'app' => $item['app'],
-			'rendered-hash' => $item['rendered-hash'],
-			'rendered-html' => $item['rendered-html'],
-			'object-type' => $item['object-type'],
-			'object' => $item['object'], 'target-type' => $item['target-type'],
-			'target' => $item['target'], 'plink' => $item['plink'],
+		$fields = ['uri' => $item['uri'], 'plink' => $item['plink'],
 			'uri-plink-hash' => hash('sha1', $item['plink']).hash('sha1', $item['uri'])];
 
+		foreach (self::CONTENT_FIELDLIST as $field) {
+			if (isset($item[$field])) {
+				$fields[$field] = $item[$field];
+				unset($item[$field]);
+			}
+		}
+
 		dba::insert('item-content', $fields, true);
 	}
 

From 89fb28ae9b735a1de0a526548cc3f7c57ce0e941 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 25 Jun 2018 04:56:32 +0000
Subject: [PATCH 04/11] Content is now stored exclusively in item-content,
 connected via "icid" field

---
 database.sql                 | 34 +++++++++++++++++++++++-----
 mod/item.php                 |  4 ++--
 src/Database/DBStructure.php |  3 +--
 src/Model/Item.php           | 43 +++++++++++++++++++++++-------------
 4 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/database.sql b/database.sql
index 9dc0bcb187..cca1163145 100644
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2018.08-dev (The Tazmans Flax-lily)
--- DB_UPDATE_VERSION 1270
+-- DB_UPDATE_VERSION 1271
 -- ------------------------------------------
 
 
@@ -477,6 +477,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 	`author-name` varchar(255) NOT NULL DEFAULT '' COMMENT 'Name of the author of this item',
 	`author-link` varchar(255) NOT NULL DEFAULT '' COMMENT 'Link to the profile page of the author of this item',
 	`author-avatar` varchar(255) NOT NULL DEFAULT '' COMMENT 'Link to the avatar picture of the author of this item',
+	`icid` int unsigned COMMENT 'Id of the item-content table entry that contains the whole item content',
 	`title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
 	`content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`body` mediumtext COMMENT 'item body content',
@@ -540,10 +541,33 @@ CREATE TABLE IF NOT EXISTS `item` (
 	 INDEX `contactid_verb` (`contact-id`,`verb`),
 	 INDEX `deleted_changed` (`deleted`,`changed`),
 	 INDEX `uid_wall_changed` (`uid`,`wall`,`changed`),
-	 INDEX `uid_eventid` (`uid`,`event-id`),
-	 INDEX `uid_authorlink` (`uid`,`author-link`(190)),
-	 INDEX `uid_ownerlink` (`uid`,`owner-link`(190))
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='All posts';
+	 INDEX `uid_eventid` (`uid`,`event-id`)
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
+
+--
+-- TABLE item-content
+--
+CREATE TABLE IF NOT EXISTS `item-content` (
+	`id` int unsigned NOT NULL auto_increment,
+	`uri` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`uri-plink-hash` char(80) NOT NULL DEFAULT '' COMMENT 'SHA-1 hash from uri and plink',
+	`title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
+	`content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`body` mediumtext COMMENT 'item body content',
+	`location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated',
+	`coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated',
+	`app` varchar(255) NOT NULL DEFAULT '' COMMENT 'application which generated this item',
+	`rendered-hash` varchar(32) NOT NULL DEFAULT '' COMMENT '',
+	`rendered-html` mediumtext COMMENT 'item.body converted to html',
+	`object-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams object type',
+	`object` text COMMENT 'JSON encoded object structure unless it is an implied object (normal post)',
+	`target-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams target type if applicable (URI)',
+	`target` text COMMENT 'JSON encoded target structure if used',
+	`plink` varchar(255) NOT NULL DEFAULT '' COMMENT 'permalink or URL to a displayable copy of the message at its source',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `uri-plink-hash` (`uri-plink-hash`),
+	 INDEX `uri` (`uri`(191))
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts';
 
 --
 -- TABLE locks
diff --git a/mod/item.php b/mod/item.php
index 4c78af6ca1..2830cae2c9 100644
--- a/mod/item.php
+++ b/mod/item.php
@@ -93,9 +93,9 @@ function item_post(App $a) {
 
 	if ($thr_parent || $thr_parent_uri) {
 		if ($thr_parent) {
-			$parent_item = dba::selectFirst('item', [], ['id' => $thr_parent]);
+			$parent_item = Item::selectFirst([], ['id' => $thr_parent]);
 		} elseif ($thr_parent_uri) {
-			$parent_item = dba::selectFirst('item', [], ['uri' => $thr_parent_uri, 'uid' => $profile_uid]);
+			$parent_item = Item::selectFirst([], ['uri' => $thr_parent_uri, 'uid' => $profile_uid]);
 		}
 
 		// if this isn't the real parent of the conversation, find it
diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php
index 19930b5e4c..e6bbf22094 100644
--- a/src/Database/DBStructure.php
+++ b/src/Database/DBStructure.php
@@ -1181,6 +1181,7 @@ class DBStructure
 						"author-name" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Name of the author of this item"],
 						"author-link" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the profile page of the author of this item"],
 						"author-avatar" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the avatar picture of the author of this item"],
+						"icid" => ["type" => "int unsigned", "relation" => ["item-content" => "id"], "comment" => "Id of the item-content table entry that contains the whole item content"],
 						"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
 						"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
 						"body" => ["type" => "mediumtext", "comment" => "item body content"],
@@ -1247,8 +1248,6 @@ class DBStructure
 						"deleted_changed" => ["deleted","changed"],
 						"uid_wall_changed" => ["uid","wall","changed"],
 						"uid_eventid" => ["uid","event-id"],
-						"uid_authorlink" => ["uid","author-link(190)"],
-						"uid_ownerlink" => ["uid","owner-link(190)"],
 						]
 				];
 		$database["item-content"] = [
diff --git a/src/Model/Item.php b/src/Model/Item.php
index 3936a3af47..40b42ea278 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -72,10 +72,10 @@ class Item extends BaseObject
 
 		// Fetch data from the item-content table whenever there is content there
 		foreach (self::CONTENT_FIELDLIST as $field) {
-			if (!empty($row['item-content-' . $field])) {
-				$row[$field] = $row['item-content-' . $field];
-				unset($row['item-content-' . $field]);
+			if (is_null($row[$field]) && !is_null($row['item-' . $field])) {
+				$row[$field] = $row['item-' . $field];
 			}
+			unset($row['item-' . $field]);
 		}
 
 		// We prefer the data from the user's contact over the public one
@@ -363,15 +363,15 @@ class Item extends BaseObject
 
 		$fields['item'] = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid',
 			'contact-id', 'owner-id', 'author-id', 'type', 'wall', 'gravity', 'extid',
-			'created', 'edited', 'commented', 'received', 'changed',
-			'title', 'body', 'app', 'verb', 'object-type', 'object', 'target-type', 'target',
+			'created', 'edited', 'commented', 'received', 'changed', 'verb',
 			'postopts', 'plink', 'resource-id', 'event-id', 'tag', 'attach', 'inform',
-			'file', 'location', 'coord', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
+			'file', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
 			'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark',
-			'unseen', 'deleted', 'origin', 'forum_mode', 'mention',
-			'rendered-hash', 'rendered-html', 'global', 'content-warning',
+			'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global',
 			'id' => 'item_id', 'network'];
 
+		$fields['item-content'] = self::CONTENT_FIELDLIST;
+
 		$fields['author'] = ['url' => 'author-link', 'name' => 'author-name',
 			'thumb' => 'author-avatar', 'nick' => 'author-nick'];
 
@@ -462,7 +462,7 @@ class Item extends BaseObject
 		}
 
 		if (strpos($sql_commands, "`item-content`.") !== false) {
-			$joins .= " LEFT JOIN `item-content` ON `item-content`.`uri` = `item`.`uri`";
+			$joins .= " LEFT JOIN `item-content` ON `item-content`.`id` = `item`.`icid`";
 		}
 
 		if ((strpos($sql_commands, "`parent-item`.") !== false) || (strpos($sql_commands, "`parent-author`.") !== false)) {
@@ -491,7 +491,7 @@ class Item extends BaseObject
 			foreach ($table_fields as $field => $select) {
 				if (empty($selected) || in_array($select, $selected)) {
 					if (in_array($select, self::CONTENT_FIELDLIST)) {
-						$selection[] = "`item-content`.`".$select."` AS `item-content-" . $select . "`";
+						$selection[] = "`item`.`".$select."` AS `item-" . $select . "`";
 					}
 					if (is_int($field)) {
 						$selection[] = "`" . $table . "`.`" . $select . "`";
@@ -579,7 +579,7 @@ class Item extends BaseObject
 		$rows = dba::affected_rows();
 
 		while ($item = dba::fetch($items)) {
-			self::updateContent($content_fields, ['uri' => $item['uri']]);
+			self::updateContent($content_fields, ['id' => $item['icid']]);
 			Term::insertFromTagFieldByItemId($item['id']);
 			Term::insertFromFileFieldByItemId($item['id']);
 			self::updateThread($item['id']);
@@ -1446,11 +1446,11 @@ class Item extends BaseObject
 	 */
 	private static function insertContent(&$item)
 	{
-		logger('Insert content for URI '.$item['uri']);
-
 		$fields = ['uri' => $item['uri'], 'plink' => $item['plink'],
 			'uri-plink-hash' => hash('sha1', $item['plink']).hash('sha1', $item['uri'])];
 
+		unset($item['plink']);
+
 		foreach (self::CONTENT_FIELDLIST as $field) {
 			if (isset($item[$field])) {
 				$fields[$field] = $item[$field];
@@ -1458,7 +1458,20 @@ class Item extends BaseObject
 			}
 		}
 
-		dba::insert('item-content', $fields, true);
+		// Do we already have this content?
+		$item_content = dba::selectFirst('item-content', ['id'], ['uri' => $item['uri']]);
+		if (DBM::is_result($item_content)) {
+			$item['icid'] = $item_content['id'];
+			logger('Assigned content for URI '.$item['uri'].' ('.$item['icid'].')');
+			return;
+		}
+
+		dba::insert('item-content', $fields);
+
+		$item['icid'] = dba::lastInsertId();
+
+		logger('Insert content for URI '.$item['uri'].' ('.$item['icid'].')');
+
 	}
 
 	/**
@@ -1481,7 +1494,7 @@ class Item extends BaseObject
 			return;
 		}
 
-		logger('Update content for URI '.$condition['uri']);
+		logger('Update content for id '.$condition['id']);
 
 		dba::update('item-content', $fields, $condition, true);
 	}

From 6dbbeaf8f667b1989cacdbf937a84061ecb0da32 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 25 Jun 2018 06:33:12 +0000
Subject: [PATCH 05/11] Remove unneeded code

---
 src/Model/Item.php | 25 +++----------------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 40b42ea278..0dd0b264ea 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1528,8 +1528,6 @@ class Item extends BaseObject
 		unset($item['wall']);
 		unset($item['origin']);
 		unset($item['starred']);
-		unset($item['rendered-hash']);
-		unset($item['rendered-html']);
 
 		$users = [];
 
@@ -1659,8 +1657,6 @@ class Item extends BaseObject
 				unset($item['mention']);
 				unset($item['origin']);
 				unset($item['starred']);
-				unset($item['rendered-hash']);
-				unset($item['rendered-html']);
 				if ($item['uri'] == $item['parent-uri']) {
 					$item['contact-id'] = Contact::getIdForURL($item['owner-link']);
 				} else {
@@ -1727,8 +1723,6 @@ class Item extends BaseObject
 		unset($item['mention']);
 		unset($item['origin']);
 		unset($item['starred']);
-		unset($item['rendered-hash']);
-		unset($item['rendered-html']);
 		$item['contact-id'] = Contact::getIdForURL($item['author-link']);
 
 		if (in_array($item['type'], ["net-comment", "wall-comment"])) {
@@ -2696,9 +2690,9 @@ EOT;
 
 	private static function updateThread($itemid, $setmention = false)
 	{
-		$fields = ['uid', 'guid', 'title', 'body', 'created', 'edited', 'commented', 'received', 'changed',
+		$fields = ['uid', 'guid', 'created', 'edited', 'commented', 'received', 'changed',
 			'wall', 'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark', 'contact-id',
-			'deleted', 'origin', 'forum_mode', 'network', 'author-id', 'owner-id', 'rendered-html', 'rendered-hash'];
+			'deleted', 'origin', 'forum_mode', 'network', 'author-id', 'owner-id'];
 		$condition = ["`id` = ? AND (`parent` = ? OR `parent` = 0)", $itemid, $itemid];
 
 		$item = dba::selectFirst('item', $fields, $condition);
@@ -2715,7 +2709,7 @@ EOT;
 		$fields = [];
 
 		foreach ($item as $field => $data) {
-			if (!in_array($field, ["guid", "title", "body", "rendered-html", "rendered-hash"])) {
+			if (!in_array($field, ["guid"])) {
 				$fields[$field] = $data;
 			}
 		}
@@ -2723,19 +2717,6 @@ EOT;
 		$result = dba::update('thread', $fields, ['iid' => $itemid]);
 
 		logger("Update thread for item ".$itemid." - guid ".$item["guid"]." - ".(int)$result, LOGGER_DEBUG);
-
-		// Updating a shadow item entry
-		$items = dba::selectFirst('item', ['id'], ['guid' => $item['guid'], 'uid' => 0]);
-
-		if (!DBM::is_result($items)) {
-			return;
-		}
-
-		$fields = ['title' => $item['title'], 'body' => $item['body'],
-			'rendered-html' => $item['rendered-html'], 'rendered-hash' => $item['rendered-hash']];
-		$result = dba::update('item', $fields, ['id' => $items['id']]);
-
-		logger("Updating public shadow for post ".$items["id"]." - guid ".$item["guid"]." Result: ".print_r($result, true), LOGGER_DEBUG);
 	}
 
 	private static function deleteThread($itemid, $itemuri = "")

From 5f3f3490061619a7545bb3a1ccd02aaab696e623 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 25 Jun 2018 06:45:05 +0000
Subject: [PATCH 06/11] "author-link" and "owner-link" aren't stored anymore in
 the item table

---
 mod/community.php  | 5 +++--
 src/Model/Item.php | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/mod/community.php b/mod/community.php
index 59be7487bd..1e6162fb8e 100644
--- a/mod/community.php
+++ b/mod/community.php
@@ -189,9 +189,10 @@ function community_content(App $a, $update = 0)
 function community_getitems($start, $itemspage, $content)
 {
 	if ($content == 'local') {
-		$r = dba::p("SELECT `item`.`uri`, `item`.`author-link` FROM `thread`
+		$r = dba::p("SELECT `item`.`uri`, `author`.`url` AS `author-link` FROM `thread`
 			INNER JOIN `user` ON `user`.`uid` = `thread`.`uid` AND NOT `user`.`hidewall`
 			INNER JOIN `item` ON `item`.`id` = `thread`.`iid`
+			INNER JOIN `contact` AS `author` ON `author`.`id`=`item`.`author-id`
 			WHERE `thread`.`visible` AND NOT `thread`.`deleted` AND NOT `thread`.`moderated`
 			AND NOT `thread`.`private` AND `thread`.`wall` AND `thread`.`origin`
 			ORDER BY `thread`.`commented` DESC LIMIT " . intval($start) . ", " . intval($itemspage)
@@ -200,7 +201,7 @@ function community_getitems($start, $itemspage, $content)
 	} elseif ($content == 'global') {
 		$r = dba::p("SELECT `uri` FROM `thread`
 				INNER JOIN `item` ON `item`.`id` = `thread`.`iid`
-		                INNER JOIN `contact` AS `author` ON `author`.`id`=`item`.`author-id`
+				INNER JOIN `contact` AS `author` ON `author`.`id`=`item`.`author-id`
 				WHERE `thread`.`uid` = 0 AND NOT `author`.`hidden` AND NOT `author`.`blocked`
 				ORDER BY `thread`.`commented` DESC LIMIT " . intval($start) . ", " . intval($itemspage));
 		return dba::inArray($r);
diff --git a/src/Model/Item.php b/src/Model/Item.php
index 0dd0b264ea..5297f0b4e4 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1075,11 +1075,12 @@ class Item extends BaseObject
 			return 0;
 		}
 
-		//unset($item['author-link']);
+		// These fields aren't stored anymore in the item table, they are fetched upon request
+		unset($item['author-link']);
 		unset($item['author-name']);
 		unset($item['author-avatar']);
 
-		//unset($item['owner-link']);
+		unset($item['owner-link']);
 		unset($item['owner-name']);
 		unset($item['owner-avatar']);
 

From c07b81c5b9c6e2f94757e25e6c139849c7b8db55 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 25 Jun 2018 16:11:27 +0000
Subject: [PATCH 07/11] Added workarounds to fetch the complete item content

---
 src/Model/Item.php | 51 +++++++++++++++++++++++++++++++++-------------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 5297f0b4e4..a093934e93 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -555,7 +555,7 @@ class Item extends BaseObject
 		// We cannot simply expand the condition to check for origin entries
 		// The condition needn't to be a simple array but could be a complex condition.
 		// And we have to execute this query before the update to ensure to fetch the same data.
-		$items = dba::select('item', ['id', 'origin', 'uri'], $condition);
+		$items = dba::select('item', ['id', 'origin', 'uri', 'plink'], $condition);
 
 		$content_fields = [];
 		foreach (self::CONTENT_FIELDLIST as $field) {
@@ -579,7 +579,10 @@ class Item extends BaseObject
 		$rows = dba::affected_rows();
 
 		while ($item = dba::fetch($items)) {
-			self::updateContent($content_fields, ['id' => $item['icid']]);
+			if (!empty($item['plink'])) {
+				$content_fields['plink'] =  $item['plink'];
+			}
+			self::updateContent($content_fields, ['uri' => $item['uri']]);
 			Term::insertFromTagFieldByItemId($item['id']);
 			Term::insertFromFileFieldByItemId($item['id']);
 			self::updateThread($item['id']);
@@ -1033,6 +1036,7 @@ class Item extends BaseObject
 
 		// When there is no content then we don't post it
 		if ($item['body'].$item['title'] == '') {
+			logger('No body, no title.');
 			return 0;
 		}
 
@@ -1460,19 +1464,15 @@ class Item extends BaseObject
 		}
 
 		// Do we already have this content?
+		if (!dba::exists('item-content', ['uri' => $item['uri']])) {
+			dba::insert('item-content', $fields, true);
+		}
+
 		$item_content = dba::selectFirst('item-content', ['id'], ['uri' => $item['uri']]);
 		if (DBM::is_result($item_content)) {
 			$item['icid'] = $item_content['id'];
-			logger('Assigned content for URI '.$item['uri'].' ('.$item['icid'].')');
-			return;
+			logger('Insert content for URI '.$item['uri'].' ('.$item['icid'].')');
 		}
-
-		dba::insert('item-content', $fields);
-
-		$item['icid'] = dba::lastInsertId();
-
-		logger('Insert content for URI '.$item['uri'].' ('.$item['icid'].')');
-
 	}
 
 	/**
@@ -1495,7 +1495,12 @@ class Item extends BaseObject
 			return;
 		}
 
-		logger('Update content for id '.$condition['id']);
+		if (!empty($item['plink'])) {
+			$fields['plink'] = $item['plink'];
+			$fields['uri-plink-hash'] = hash('sha1', $item['plink']).hash('sha1', $condition['uri']);
+		}
+
+		logger('Update content for URI '.$condition['uri']);
 
 		dba::update('item-content', $fields, $condition, true);
 	}
@@ -1523,6 +1528,15 @@ class Item extends BaseObject
 			return;
 		}
 
+		$fields = self::CONTENT_FIELDLIST;
+		$fields[] = 'author-link';
+		$fields[] = 'owner-link';
+
+		$content = self::selectFirst($fields, ['id' => $itemid]);
+		if (DBM::is_result($content)) {
+			$item = array_merge($item, $content);
+		}
+
 		unset($item['id']);
 		unset($item['parent']);
 		unset($item['mention']);
@@ -1646,6 +1660,15 @@ class Item extends BaseObject
 
 		$item = dba::selectFirst('item', [], ['id' => $itemid]);
 
+		$fields = self::CONTENT_FIELDLIST;
+		$fields[] = 'author-link';
+		$fields[] = 'owner-link';
+
+		$content = self::selectFirst($fields, ['id' => $itemid]);
+		if (DBM::is_result($content)) {
+			$item = array_merge($item, $content);
+		}
+
 		if (DBM::is_result($item) && ($item["allow_cid"] == '') && ($item["allow_gid"] == '') &&
 			($item["deny_cid"] == '') && ($item["deny_gid"] == '')) {
 
@@ -1659,9 +1682,9 @@ class Item extends BaseObject
 				unset($item['origin']);
 				unset($item['starred']);
 				if ($item['uri'] == $item['parent-uri']) {
-					$item['contact-id'] = Contact::getIdForURL($item['owner-link']);
+					$item['contact-id'] = $item['owner-id'];
 				} else {
-					$item['contact-id'] = Contact::getIdForURL($item['author-link']);
+					$item['contact-id'] = $item['author-id'];
 				}
 
 				if (in_array($item['type'], ["net-comment", "wall-comment"])) {

From 89f0eec261b2fff766cc1867e5aa8a9c29e7903d Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 25 Jun 2018 18:49:36 +0000
Subject: [PATCH 08/11] Use a constant fieldlist

---
 src/Model/Item.php | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index a093934e93..3b46e7f15d 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -56,10 +56,24 @@ class Item extends BaseObject
 			'author-id', 'author-link', 'owner-link', 'contact-uid',
 			'signed_text', 'signature', 'signer'];
 
+	// Field list for "item-content" table that is mixed with the item table
 	const CONTENT_FIELDLIST = ['title', 'content-warning', 'body', 'location',
 			'coord', 'app', 'rendered-hash', 'rendered-html',
 			'object-type', 'object', 'target-type', 'target'];
 
+	// All fields in the item table
+	const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid',
+			'contact-id', 'type', 'wall', 'gravity', 'extid', 'icid',
+			'created', 'edited', 'commented', 'received', 'changed', 'verb',
+			'postopts', 'plink', 'resource-id', 'event-id', 'tag', 'attach', 'inform',
+			'file', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
+			'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark',
+			'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global', 'network',
+			'title', 'content-warning', 'body', 'location', 'coord', 'app',
+			'rendered-hash', 'rendered-html', 'object-type', 'object', 'target-type', 'target',
+			'author-id', 'author-link', 'author-name', 'author-avatar',
+			'owner-id', 'owner-link', 'owner-name', 'owner-avatar'];
+
 	/**
 	 * @brief Fetch a single item row
 	 *
@@ -1523,20 +1537,11 @@ class Item extends BaseObject
 		$condition = ['id' => $itemid, 'uid' => 0,
 			'network' => [NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, ""],
 			'visible' => true, 'deleted' => false, 'moderated' => false, 'private' => false];
-		$item = dba::selectFirst('item', [], ['id' => $itemid]);
+		$item = self::selectFirst(self::ITEM_FIELDLIST, ['id' => $itemid]);
 		if (!DBM::is_result($item)) {
 			return;
 		}
 
-		$fields = self::CONTENT_FIELDLIST;
-		$fields[] = 'author-link';
-		$fields[] = 'owner-link';
-
-		$content = self::selectFirst($fields, ['id' => $itemid]);
-		if (DBM::is_result($content)) {
-			$item = array_merge($item, $content);
-		}
-
 		unset($item['id']);
 		unset($item['parent']);
 		unset($item['mention']);
@@ -1658,16 +1663,7 @@ class Item extends BaseObject
 			return;
 		}
 
-		$item = dba::selectFirst('item', [], ['id' => $itemid]);
-
-		$fields = self::CONTENT_FIELDLIST;
-		$fields[] = 'author-link';
-		$fields[] = 'owner-link';
-
-		$content = self::selectFirst($fields, ['id' => $itemid]);
-		if (DBM::is_result($content)) {
-			$item = array_merge($item, $content);
-		}
+		$item = self::selectFirst(self::ITEM_FIELDLIST, ['id' => $itemid]);
 
 		if (DBM::is_result($item) && ($item["allow_cid"] == '') && ($item["allow_gid"] == '') &&
 			($item["deny_cid"] == '') && ($item["deny_gid"] == '')) {

From 6b568bb8faf90a9402b6b5c7ffb68d2ebd1c6d6e Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 25 Jun 2018 20:23:32 +0000
Subject: [PATCH 09/11] Delete of item-content should work now

---
 database.sql                 | 3 ++-
 src/Database/DBStructure.php | 1 +
 src/Worker/Expire.php        | 5 ++++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/database.sql b/database.sql
index cca1163145..9711ace803 100644
--- a/database.sql
+++ b/database.sql
@@ -541,7 +541,8 @@ CREATE TABLE IF NOT EXISTS `item` (
 	 INDEX `contactid_verb` (`contact-id`,`verb`),
 	 INDEX `deleted_changed` (`deleted`,`changed`),
 	 INDEX `uid_wall_changed` (`uid`,`wall`,`changed`),
-	 INDEX `uid_eventid` (`uid`,`event-id`)
+	 INDEX `uid_eventid` (`uid`,`event-id`),
+	 INDEX `icid` (`icid`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
 
 --
diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php
index e6bbf22094..eaf54cee23 100644
--- a/src/Database/DBStructure.php
+++ b/src/Database/DBStructure.php
@@ -1248,6 +1248,7 @@ class DBStructure
 						"deleted_changed" => ["deleted","changed"],
 						"uid_wall_changed" => ["uid","wall","changed"],
 						"uid_eventid" => ["uid","event-id"],
+						"icid" => ["icid"],
 						]
 				];
 		$database["item-content"] = [
diff --git a/src/Worker/Expire.php b/src/Worker/Expire.php
index cf6e78cb72..713bfa25e0 100644
--- a/src/Worker/Expire.php
+++ b/src/Worker/Expire.php
@@ -26,9 +26,12 @@ class Expire {
 		if ($param == 'delete') {
 			logger('Delete expired items', LOGGER_DEBUG);
 			// physically remove anything that has been deleted for more than two months
-			$r = dba::p("SELECT `id` FROM `item` WHERE `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY");
+			$r = dba::p("SELECT `id`, `icid` FROM `item` WHERE `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY");
 			while ($row = dba::fetch($r)) {
 				dba::delete('item', ['id' => $row['id']]);
+				if (!dba::exists('item', ['icid' => $row['icid']])) {
+					dba::delete('item-content', ['id' => $row['icid']]);
+				}
 			}
 			dba::close($r);
 

From 7aa880543f0731db383109ff735a7831f7ad4e16 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 25 Jun 2018 20:38:32 +0000
Subject: [PATCH 10/11] Fix shadow posts

---
 src/Model/Item.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 3b46e7f15d..4847a1171f 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1705,7 +1705,7 @@ class Item extends BaseObject
 	 */
 	public static function addShadowPost($itemid)
 	{
-		$item = dba::selectFirst('item', [], ['id' => $itemid]);
+		$item = self::selectFirst(self::ITEM_FIELDLIST, ['id' => $itemid]);
 		if (!DBM::is_result($item)) {
 			return;
 		}

From f74a54b9bf2807b7830831e86cbbd0dfe3c7734f Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 26 Jun 2018 03:58:12 +0000
Subject: [PATCH 11/11] Standard stuff

---
 src/Model/Item.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 4847a1171f..6fe4a575dd 100644
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -1485,7 +1485,7 @@ class Item extends BaseObject
 		$item_content = dba::selectFirst('item-content', ['id'], ['uri' => $item['uri']]);
 		if (DBM::is_result($item_content)) {
 			$item['icid'] = $item_content['id'];
-			logger('Insert content for URI '.$item['uri'].' ('.$item['icid'].')');
+			logger('Insert content for URI ' . $item['uri'] . ' (' . $item['icid'] . ')');
 		}
 	}
 
@@ -1511,10 +1511,10 @@ class Item extends BaseObject
 
 		if (!empty($item['plink'])) {
 			$fields['plink'] = $item['plink'];
-			$fields['uri-plink-hash'] = hash('sha1', $item['plink']).hash('sha1', $condition['uri']);
+			$fields['uri-plink-hash'] = hash('sha1', $item['plink']) . hash('sha1', $condition['uri']);
 		}
 
-		logger('Update content for URI '.$condition['uri']);
+		logger('Update content for URI ' . $condition['uri']);
 
 		dba::update('item-content', $fields, $condition, true);
 	}