We now store the uri data in a separate table (#5560)

* We now have a item-uri table

* Fixing line endings

* New item uri table

* Rescued ItemURI.php file

* Reverting some git problem

* Corrected the dbstructure file

* Updated database.sql / we now store content in the new id fields
This commit is contained in:
Michael Vogel 2018-08-05 13:09:59 +02:00 committed by Tobias Diekershoff
parent ffe16be7d6
commit fa46e97f27
5 changed files with 103 additions and 5 deletions

View file

@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM', 'Friendica');
define('FRIENDICA_CODENAME', 'The Tazmans Flax-lily'); define('FRIENDICA_CODENAME', 'The Tazmans Flax-lily');
define('FRIENDICA_VERSION', '2018.08-dev'); define('FRIENDICA_VERSION', '2018.08-dev');
define('DFRN_PROTOCOL_VERSION', '2.23'); define('DFRN_PROTOCOL_VERSION', '2.23');
define('DB_UPDATE_VERSION', 1280); define('DB_UPDATE_VERSION', 1281);
define('NEW_UPDATE_ROUTINE_VERSION', 1170); define('NEW_UPDATE_ROUTINE_VERSION', 1170);
/** /**

View file

@ -211,7 +211,7 @@
"conversation": { "conversation": {
"comment": "Raw data and structure information for messages", "comment": "Raw data and structure information for messages",
"fields": { "fields": {
"item-uri": {"type": "varbinary(255)", "not null": "1", "primary": "1", "comment": "URI of the item"}, "item-uri": {"type": "varbinary(255)", "not null": "1", "primary": "1", "comment": "Original URI of the item - unrelated to the table with the same name"},
"reply-to-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "URI to which this item is a reply"}, "reply-to-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "URI to which this item is a reply"},
"conversation-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation URI"}, "conversation-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation URI"},
"conversation-href": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation link"}, "conversation-href": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation link"},
@ -471,10 +471,13 @@
"id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}}, "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
"guid": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "A unique identifier for this item"}, "guid": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "A unique identifier for this item"},
"uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""}, "uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""},
"uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
"uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"}, "uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
"parent": {"type": "int unsigned", "not null": "1", "default": "0", "relation": {"item": "id"}, "comment": "item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item"}, "parent": {"type": "int unsigned", "not null": "1", "default": "0", "relation": {"item": "id"}, "comment": "item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item"},
"parent-uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "uri of the parent to this item"}, "parent-uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "uri of the parent to this item"},
"parent-uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table that contains the parent uri"},
"thr-parent": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri"}, "thr-parent": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri"},
"thr-parent-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table that contains the thread parent uri"},
"created": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Creation timestamp."}, "created": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Creation timestamp."},
"edited": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last edit (default is created)"}, "edited": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last edit (default is created)"},
"commented": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last comment/reply to this item"}, "commented": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last comment/reply to this item"},
@ -571,6 +574,7 @@
"fields": { "fields": {
"id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}}, "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
"uri": {"type": "varchar(255)", "comment": ""}, "uri": {"type": "varchar(255)", "comment": ""},
"uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
"uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"}, "uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
"activity": {"type": "smallint unsigned", "not null": "1", "default": "0", "comment": ""} "activity": {"type": "smallint unsigned", "not null": "1", "default": "0", "comment": ""}
}, },
@ -585,6 +589,7 @@
"fields": { "fields": {
"id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}}, "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
"uri": {"type": "varchar(255)", "comment": ""}, "uri": {"type": "varchar(255)", "comment": ""},
"uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
"uri-plink-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"}, "uri-plink-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
"title": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "item title"}, "title": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "item title"},
"content-warning": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""}, "content-warning": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""},
@ -619,6 +624,19 @@
"PRIMARY": ["iid"] "PRIMARY": ["iid"]
} }
}, },
"item-uri": {
"comment": "URI and GUID for items",
"fields": {
"id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1"},
"uri": {"type": "varbinary(255)", "not null": "1", "comment": "URI of an item"},
"guid": {"type": "varbinary(255)", "comment": "A unique identifier for an item"}
},
"indexes": {
"PRIMARY": ["id"],
"uri": ["UNIQUE", "uri"],
"guid": ["guid"]
}
},
"locks": { "locks": {
"comment": "", "comment": "",
"fields": { "fields": {

View file

@ -1,6 +1,6 @@
-- ------------------------------------------ -- ------------------------------------------
-- Friendica 2018.08-dev (The Tazmans Flax-lily) -- Friendica 2018.08-dev (The Tazmans Flax-lily)
-- DB_UPDATE_VERSION 1280 -- DB_UPDATE_VERSION 1281
-- ------------------------------------------ -- ------------------------------------------
@ -208,7 +208,7 @@ CREATE TABLE IF NOT EXISTS `conv` (
-- TABLE conversation -- TABLE conversation
-- --
CREATE TABLE IF NOT EXISTS `conversation` ( CREATE TABLE IF NOT EXISTS `conversation` (
`item-uri` varbinary(255) NOT NULL COMMENT 'URI of the item', `item-uri` varbinary(255) NOT NULL COMMENT 'Original URI of the item - unrelated to the table with the same name',
`reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply', `reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply',
`conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI', `conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI',
`conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link', `conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link',
@ -455,10 +455,13 @@ CREATE TABLE IF NOT EXISTS `item` (
`id` int unsigned NOT NULL auto_increment, `id` int unsigned NOT NULL auto_increment,
`guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'A unique identifier for this item', `guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'A unique identifier for this item',
`uri` varchar(255) NOT NULL DEFAULT '' COMMENT '', `uri` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
`uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
`parent` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item', `parent` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item',
`parent-uri` varchar(255) NOT NULL DEFAULT '' COMMENT 'uri of the parent to this item', `parent-uri` varchar(255) NOT NULL DEFAULT '' COMMENT 'uri of the parent to this item',
`parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri',
`thr-parent` varchar(255) NOT NULL DEFAULT '' COMMENT 'If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri', `thr-parent` varchar(255) NOT NULL DEFAULT '' COMMENT 'If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri',
`thr-parent-id` int unsigned COMMENT 'Id of the item-uri table that contains the thread parent uri',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.',
`edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)', `edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)',
`commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last comment/reply to this item', `commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last comment/reply to this item',
@ -554,6 +557,7 @@ CREATE TABLE IF NOT EXISTS `item` (
CREATE TABLE IF NOT EXISTS `item-activity` ( CREATE TABLE IF NOT EXISTS `item-activity` (
`id` int unsigned NOT NULL auto_increment, `id` int unsigned NOT NULL auto_increment,
`uri` varchar(255) COMMENT '', `uri` varchar(255) COMMENT '',
`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
`uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
`activity` smallint unsigned NOT NULL DEFAULT 0 COMMENT '', `activity` smallint unsigned NOT NULL DEFAULT 0 COMMENT '',
PRIMARY KEY(`id`), PRIMARY KEY(`id`),
@ -567,6 +571,7 @@ CREATE TABLE IF NOT EXISTS `item-activity` (
CREATE TABLE IF NOT EXISTS `item-content` ( CREATE TABLE IF NOT EXISTS `item-content` (
`id` int unsigned NOT NULL auto_increment, `id` int unsigned NOT NULL auto_increment,
`uri` varchar(255) COMMENT '', `uri` varchar(255) COMMENT '',
`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
`uri-plink-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `uri-plink-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
`title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title', `title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
`content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '', `content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
@ -598,6 +603,18 @@ CREATE TABLE IF NOT EXISTS `item-delivery-data` (
PRIMARY KEY(`iid`) PRIMARY KEY(`iid`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items'; ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items';
--
-- TABLE item-uri
--
CREATE TABLE IF NOT EXISTS `item-uri` (
`id` int unsigned NOT NULL auto_increment,
`uri` varbinary(255) NOT NULL COMMENT 'URI of an item',
`guid` varbinary(255) COMMENT 'A unique identifier for an item',
PRIMARY KEY(`id`),
UNIQUE INDEX `uri` (`uri`),
INDEX `guid` (`guid`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='URI and GUID for items';
-- --
-- TABLE locks -- TABLE locks
-- --

View file

@ -17,6 +17,7 @@ use Friendica\Core\Worker;
use Friendica\Database\DBA; use Friendica\Database\DBA;
use Friendica\Model\Contact; use Friendica\Model\Contact;
use Friendica\Model\PermissionSet; use Friendica\Model\PermissionSet;
use Friendica\Model\ItemURI;
use Friendica\Object\Image; use Friendica\Object\Image;
use Friendica\Protocol\Diaspora; use Friendica\Protocol\Diaspora;
use Friendica\Protocol\OStatus; use Friendica\Protocol\OStatus;
@ -1253,6 +1254,9 @@ class Item extends BaseObject
$item['guid'] = self::guid($item, $notify); $item['guid'] = self::guid($item, $notify);
$item['uri'] = notags(trim(defaults($item, 'uri', self::newURI($item['uid'], $item['guid'])))); $item['uri'] = notags(trim(defaults($item, 'uri', self::newURI($item['uid'], $item['guid']))));
// Store URI data
$item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]);
// Store conversation data // Store conversation data
$item = Conversation::insert($item); $item = Conversation::insert($item);
@ -1565,6 +1569,9 @@ class Item extends BaseObject
} }
} }
$item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']);
$item['thr-parent-id'] = ItemURI::getIdByURI($item['thr-parent']);
$condition = ["`uri` = ? AND `network` IN (?, ?) AND `uid` = ?", $condition = ["`uri` = ? AND `network` IN (?, ?) AND `uid` = ?",
$item['uri'], $item['network'], NETWORK_DFRN, $item['uid']]; $item['uri'], $item['network'], NETWORK_DFRN, $item['uid']];
if (self::exists($condition)) { if (self::exists($condition)) {

56
src/Model/ItemURI.php Normal file
View file

@ -0,0 +1,56 @@
<?php
/**
* @file src/Model/ItemURI.php
*/
namespace Friendica\Model;
use Friendica\BaseObject;
use Friendica\Database\DBA;
require_once 'boot.php';
class ItemURI extends BaseObject
{
/**
* @brief Insert an item-uri record and return its id
*
* @param array $fields Item-uri fields
*
* @return integer item-uri id
*/
public static function insert($fields)
{
if (!DBA::exists('item-uri', ['uri' => $fields['uri']])) {
DBA::insert('item-uri', $fields, true);
}
$itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $fields['uri']]);
if (!DBA::isResult($itemuri)) {
// This shouldn't happen
return null;
}
return $itemuri['id'];
}
/**
* @brief Searched for an id of a given uri. Adds it, if not existing yet.
*
* @param string $uri
*
* @return integer item-uri id
*/
public static function getIdByURI($uri)
{
$itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $uri]);
if (!DBA::isResult($itemuri)) {
return self::insert(['uri' => $uri]);
}
return $itemuri['id'];
}
}