We now store the uri data in a separate table (#5560)

* We now have a item-uri table

* Fixing line endings

* New item uri table

* Rescued ItemURI.php file

* Reverting some git problem

* Corrected the dbstructure file

* Updated database.sql / we now store content in the new id fields

boot.php

@@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'The Tazmans Flax-lily');
 define('FRIENDICA_VERSION',      '2018.08-dev');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
-define('DB_UPDATE_VERSION',      1280);
+define('DB_UPDATE_VERSION',      1281);
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
 /**

config/dbstructure.json

@@ -211,7 +211,7 @@
 	"conversation": {
 		"comment": "Raw data and structure information for messages",
 		"fields": {
-			"item-uri": {"type": "varbinary(255)", "not null": "1", "primary": "1", "comment": "URI of the item"},
+			"item-uri": {"type": "varbinary(255)", "not null": "1", "primary": "1", "comment": "Original URI of the item - unrelated to the table with the same name"},
 			"reply-to-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "URI to which this item is a reply"},
 			"conversation-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation URI"},
 			"conversation-href": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation link"},
@@ -471,10 +471,13 @@
 			"id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
 			"guid": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "A unique identifier for this item"},
 			"uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""},
+			"uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
 			"uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
 			"parent": {"type": "int unsigned", "not null": "1", "default": "0", "relation": {"item": "id"}, "comment": "item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item"},
 			"parent-uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "uri of the parent to this item"},
+			"parent-uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table that contains the parent uri"},
 			"thr-parent": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri"},
+			"thr-parent-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table that contains the thread parent uri"},
 			"created": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Creation timestamp."},
 			"edited": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last edit (default is created)"},
 			"commented": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last comment/reply to this item"},
@@ -571,6 +574,7 @@
 		"fields": {
 			"id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
 			"uri": {"type": "varchar(255)", "comment": ""},
+			"uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
 			"uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
 			"activity": {"type": "smallint unsigned", "not null": "1", "default": "0", "comment": ""}
 		},
@@ -585,6 +589,7 @@
 		"fields": {
 			"id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
 			"uri": {"type": "varchar(255)", "comment": ""},
+			"uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
 			"uri-plink-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
 			"title": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "item title"},
 			"content-warning": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""},
@@ -619,6 +624,19 @@
 			"PRIMARY": ["iid"]
 		}
 	},
+	"item-uri": {
+		"comment": "URI and GUID for items",
+		"fields": {
+			"id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1"},
+			"uri": {"type": "varbinary(255)", "not null": "1", "comment": "URI of an item"},
+			"guid": {"type": "varbinary(255)", "comment": "A unique identifier for an item"}
+		},
+		"indexes": {
+			"PRIMARY": ["id"],
+			"uri": ["UNIQUE", "uri"],
+			"guid": ["guid"]
+		}
+	},
 	"locks": {
 		"comment": "",
 		"fields": {

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2018.08-dev (The Tazmans Flax-lily)
--- DB_UPDATE_VERSION 1280
+-- DB_UPDATE_VERSION 1281
 -- ------------------------------------------
 
 
@@ -208,7 +208,7 @@ CREATE TABLE IF NOT EXISTS `conv` (
 -- TABLE conversation
 --
 CREATE TABLE IF NOT EXISTS `conversation` (
-	`item-uri` varbinary(255) NOT NULL COMMENT 'URI of the item',
+	`item-uri` varbinary(255) NOT NULL COMMENT 'Original URI of the item - unrelated to the table with the same name',
 	`reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply',
 	`conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI',
 	`conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link',
@@ -455,10 +455,13 @@ CREATE TABLE IF NOT EXISTS `item` (
 	`id` int unsigned NOT NULL auto_increment,
 	`guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'A unique identifier for this item',
 	`uri` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
 	`uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
 	`parent` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item',
 	`parent-uri` varchar(255) NOT NULL DEFAULT '' COMMENT 'uri of the parent to this item',
+	`parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri',
 	`thr-parent` varchar(255) NOT NULL DEFAULT '' COMMENT 'If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri',
+	`thr-parent-id` int unsigned COMMENT 'Id of the item-uri table that contains the thread parent uri',
 	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.',
 	`edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)',
 	`commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last comment/reply to this item',
@@ -554,6 +557,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 CREATE TABLE IF NOT EXISTS `item-activity` (
 	`id` int unsigned NOT NULL auto_increment,
 	`uri` varchar(255) COMMENT '',
+	`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
 	`uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
 	`activity` smallint unsigned NOT NULL DEFAULT 0 COMMENT '',
 	 PRIMARY KEY(`id`),
@@ -567,6 +571,7 @@ CREATE TABLE IF NOT EXISTS `item-activity` (
 CREATE TABLE IF NOT EXISTS `item-content` (
 	`id` int unsigned NOT NULL auto_increment,
 	`uri` varchar(255) COMMENT '',
+	`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
 	`uri-plink-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
 	`title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
 	`content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
@@ -598,6 +603,18 @@ CREATE TABLE IF NOT EXISTS `item-delivery-data` (
 	 PRIMARY KEY(`iid`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items';
 
+--
+-- TABLE item-uri
+--
+CREATE TABLE IF NOT EXISTS `item-uri` (
+	`id` int unsigned NOT NULL auto_increment,
+	`uri` varbinary(255) NOT NULL COMMENT 'URI of an item',
+	`guid` varbinary(255) COMMENT 'A unique identifier for an item',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `uri` (`uri`),
+	 INDEX `guid` (`guid`)
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='URI and GUID for items';
+
 --
 -- TABLE locks
 --

src/Model/Item.php

@@ -17,6 +17,7 @@ use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\Model\Contact;
 use Friendica\Model\PermissionSet;
+use Friendica\Model\ItemURI;
 use Friendica\Object\Image;
 use Friendica\Protocol\Diaspora;
 use Friendica\Protocol\OStatus;
@@ -1253,6 +1254,9 @@ class Item extends BaseObject
 		$item['guid'] = self::guid($item, $notify);
 		$item['uri'] = notags(trim(defaults($item, 'uri', self::newURI($item['uid'], $item['guid']))));
 
+		// Store URI data
+		$item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]);
+
 		// Store conversation data
 		$item = Conversation::insert($item);
 
@@ -1565,6 +1569,9 @@ class Item extends BaseObject
 			}
 		}
 
+		$item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']);
+		$item['thr-parent-id'] = ItemURI::getIdByURI($item['thr-parent']);
+
 		$condition = ["`uri` = ? AND `network` IN (?, ?) AND `uid` = ?",
 			$item['uri'], $item['network'], NETWORK_DFRN, $item['uid']];
 		if (self::exists($condition)) {

src/Model/ItemURI.php

@@ -0,0 +1,56 @@
+<?php
+
+/**
+ * @file src/Model/ItemURI.php
+ */
+
+namespace Friendica\Model;
+
+use Friendica\BaseObject;
+use Friendica\Database\DBA;
+
+require_once 'boot.php';
+
+class ItemURI extends BaseObject
+{
+	/**
+	 * @brief Insert an item-uri record and return its id
+	 *
+	 * @param array $fields Item-uri fields
+	 *
+	 * @return integer item-uri id
+	 */
+	public static function insert($fields)
+	{
+		if (!DBA::exists('item-uri', ['uri' => $fields['uri']])) {
+			DBA::insert('item-uri', $fields, true);
+		}
+
+		$itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $fields['uri']]);
+
+		if (!DBA::isResult($itemuri)) {
+			// This shouldn't happen
+			return null;
+		}
+
+		return $itemuri['id'];
+	}
+
+	/**
+	 * @brief Searched for an id of a given uri. Adds it, if not existing yet.
+	 *
+	 * @param string $uri
+	 *
+	 * @return integer item-uri id
+	 */
+	public static function getIdByURI($uri)
+	{
+		$itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $uri]);
+
+		if (!DBA::isResult($itemuri)) {
+			return self::insert(['uri' => $uri]);
+		}
+
+		return $itemuri['id'];
+	}
+}