From fa46e97f2799d19387ccf9ae2e1b0ff6d66efd1a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 5 Aug 2018 13:09:59 +0200 Subject: [PATCH] We now store the uri data in a separate table (#5560) * We now have a item-uri table * Fixing line endings * New item uri table * Rescued ItemURI.php file * Reverting some git problem * Corrected the dbstructure file * Updated database.sql / we now store content in the new id fields --- boot.php | 2 +- config/dbstructure.json | 22 ++++++++++++++-- database.sql | 21 ++++++++++++++-- src/Model/Item.php | 7 ++++++ src/Model/ItemURI.php | 56 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 src/Model/ItemURI.php diff --git a/boot.php b/boot.php index f660b65b51..b8149684c3 100644 --- a/boot.php +++ b/boot.php @@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM', 'Friendica'); define('FRIENDICA_CODENAME', 'The Tazmans Flax-lily'); define('FRIENDICA_VERSION', '2018.08-dev'); define('DFRN_PROTOCOL_VERSION', '2.23'); -define('DB_UPDATE_VERSION', 1280); +define('DB_UPDATE_VERSION', 1281); define('NEW_UPDATE_ROUTINE_VERSION', 1170); /** diff --git a/config/dbstructure.json b/config/dbstructure.json index fa75133327..fce2fcf930 100644 --- a/config/dbstructure.json +++ b/config/dbstructure.json @@ -211,7 +211,7 @@ "conversation": { "comment": "Raw data and structure information for messages", "fields": { - "item-uri": {"type": "varbinary(255)", "not null": "1", "primary": "1", "comment": "URI of the item"}, + "item-uri": {"type": "varbinary(255)", "not null": "1", "primary": "1", "comment": "Original URI of the item - unrelated to the table with the same name"}, "reply-to-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "URI to which this item is a reply"}, "conversation-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation URI"}, "conversation-href": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation link"}, @@ -471,10 +471,13 @@ "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}}, "guid": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "A unique identifier for this item"}, "uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""}, + "uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"}, "uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"}, "parent": {"type": "int unsigned", "not null": "1", "default": "0", "relation": {"item": "id"}, "comment": "item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item"}, "parent-uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "uri of the parent to this item"}, + "parent-uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table that contains the parent uri"}, "thr-parent": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri"}, + "thr-parent-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table that contains the thread parent uri"}, "created": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Creation timestamp."}, "edited": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last edit (default is created)"}, "commented": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last comment/reply to this item"}, @@ -571,6 +574,7 @@ "fields": { "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}}, "uri": {"type": "varchar(255)", "comment": ""}, + "uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"}, "uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"}, "activity": {"type": "smallint unsigned", "not null": "1", "default": "0", "comment": ""} }, @@ -585,6 +589,7 @@ "fields": { "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}}, "uri": {"type": "varchar(255)", "comment": ""}, + "uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"}, "uri-plink-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"}, "title": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "item title"}, "content-warning": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""}, @@ -619,6 +624,19 @@ "PRIMARY": ["iid"] } }, + "item-uri": { + "comment": "URI and GUID for items", + "fields": { + "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1"}, + "uri": {"type": "varbinary(255)", "not null": "1", "comment": "URI of an item"}, + "guid": {"type": "varbinary(255)", "comment": "A unique identifier for an item"} + }, + "indexes": { + "PRIMARY": ["id"], + "uri": ["UNIQUE", "uri"], + "guid": ["guid"] + } + }, "locks": { "comment": "", "fields": { @@ -1247,4 +1265,4 @@ "done_executed": ["done", "executed"] } } -} \ No newline at end of file +} diff --git a/database.sql b/database.sql index 6dc3b407bf..5289328462 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2018.08-dev (The Tazmans Flax-lily) --- DB_UPDATE_VERSION 1280 +-- DB_UPDATE_VERSION 1281 -- ------------------------------------------ @@ -208,7 +208,7 @@ CREATE TABLE IF NOT EXISTS `conv` ( -- TABLE conversation -- CREATE TABLE IF NOT EXISTS `conversation` ( - `item-uri` varbinary(255) NOT NULL COMMENT 'URI of the item', + `item-uri` varbinary(255) NOT NULL COMMENT 'Original URI of the item - unrelated to the table with the same name', `reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply', `conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI', `conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link', @@ -455,10 +455,13 @@ CREATE TABLE IF NOT EXISTS `item` ( `id` int unsigned NOT NULL auto_increment, `guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'A unique identifier for this item', `uri` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri', `uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `parent` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item', `parent-uri` varchar(255) NOT NULL DEFAULT '' COMMENT 'uri of the parent to this item', + `parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri', `thr-parent` varchar(255) NOT NULL DEFAULT '' COMMENT 'If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri', + `thr-parent-id` int unsigned COMMENT 'Id of the item-uri table that contains the thread parent uri', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.', `edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)', `commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last comment/reply to this item', @@ -554,6 +557,7 @@ CREATE TABLE IF NOT EXISTS `item` ( CREATE TABLE IF NOT EXISTS `item-activity` ( `id` int unsigned NOT NULL auto_increment, `uri` varchar(255) COMMENT '', + `uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri', `uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `activity` smallint unsigned NOT NULL DEFAULT 0 COMMENT '', PRIMARY KEY(`id`), @@ -567,6 +571,7 @@ CREATE TABLE IF NOT EXISTS `item-activity` ( CREATE TABLE IF NOT EXISTS `item-content` ( `id` int unsigned NOT NULL auto_increment, `uri` varchar(255) COMMENT '', + `uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri', `uri-plink-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title', `content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '', @@ -598,6 +603,18 @@ CREATE TABLE IF NOT EXISTS `item-delivery-data` ( PRIMARY KEY(`iid`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items'; +-- +-- TABLE item-uri +-- +CREATE TABLE IF NOT EXISTS `item-uri` ( + `id` int unsigned NOT NULL auto_increment, + `uri` varbinary(255) NOT NULL COMMENT 'URI of an item', + `guid` varbinary(255) COMMENT 'A unique identifier for an item', + PRIMARY KEY(`id`), + UNIQUE INDEX `uri` (`uri`), + INDEX `guid` (`guid`) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='URI and GUID for items'; + -- -- TABLE locks -- diff --git a/src/Model/Item.php b/src/Model/Item.php index 3fe69fb127..84c51a2bf0 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -17,6 +17,7 @@ use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\Model\Contact; use Friendica\Model\PermissionSet; +use Friendica\Model\ItemURI; use Friendica\Object\Image; use Friendica\Protocol\Diaspora; use Friendica\Protocol\OStatus; @@ -1253,6 +1254,9 @@ class Item extends BaseObject $item['guid'] = self::guid($item, $notify); $item['uri'] = notags(trim(defaults($item, 'uri', self::newURI($item['uid'], $item['guid'])))); + // Store URI data + $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]); + // Store conversation data $item = Conversation::insert($item); @@ -1565,6 +1569,9 @@ class Item extends BaseObject } } + $item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']); + $item['thr-parent-id'] = ItemURI::getIdByURI($item['thr-parent']); + $condition = ["`uri` = ? AND `network` IN (?, ?) AND `uid` = ?", $item['uri'], $item['network'], NETWORK_DFRN, $item['uid']]; if (self::exists($condition)) { diff --git a/src/Model/ItemURI.php b/src/Model/ItemURI.php new file mode 100644 index 0000000000..559babb762 --- /dev/null +++ b/src/Model/ItemURI.php @@ -0,0 +1,56 @@ + $fields['uri']])) { + DBA::insert('item-uri', $fields, true); + } + + $itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $fields['uri']]); + + if (!DBA::isResult($itemuri)) { + // This shouldn't happen + return null; + } + + return $itemuri['id']; + } + + /** + * @brief Searched for an id of a given uri. Adds it, if not existing yet. + * + * @param string $uri + * + * @return integer item-uri id + */ + public static function getIdByURI($uri) + { + $itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $uri]); + + if (!DBA::isResult($itemuri)) { + return self::insert(['uri' => $uri]); + } + + return $itemuri['id']; + } +}