nupplaPhil/friendica
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #9890 from annando/post-content

Browse source 
New table "post-content"
This commit is contained in:
Hypolite Petovan 2021-01-30 18:01:13 -05:00 committed by GitHub
commit 9885c1f128
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 518 additions and 527 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

146
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------ -- ------------------------------------------
-- Friendica 2021.03-dev (Red Hot Poker) -- Friendica 2021.03-dev (Red Hot Poker)
-- DB_UPDATE_VERSION 1395 -- DB_UPDATE_VERSION 1396
-- ------------------------------------------ -- ------------------------------------------
@ -797,55 +797,6 @@ CREATE TABLE IF NOT EXISTS `item` (
FOREIGN KEY (`psid`) REFERENCES `permissionset` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT FOREIGN KEY (`psid`) REFERENCES `permissionset` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts'; ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
--
-- TABLE item-activity
--
CREATE TABLE IF NOT EXISTS `item-activity` (
`id` int unsigned NOT NULL auto_increment,
`uri` varchar(255) COMMENT '',
`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
`uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
`activity` smallint unsigned NOT NULL DEFAULT 0 COMMENT '',
PRIMARY KEY(`id`),
UNIQUE INDEX `uri-hash` (`uri-hash`),
INDEX `uri` (`uri`(191)),
INDEX `uri-id` (`uri-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Activities for items';
--
-- TABLE item-content
--
CREATE TABLE IF NOT EXISTS `item-content` (
`id` int unsigned NOT NULL auto_increment,
`uri` varchar(255) COMMENT '',
`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
`uri-plink-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
`title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
`content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`body` mediumtext COMMENT 'item body content',
`raw-body` mediumtext COMMENT 'Body without embedded media links',
`location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated',
`coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated',
`language` text COMMENT 'Language information about this post',
`app` varchar(255) NOT NULL DEFAULT '' COMMENT 'application which generated this item',
`rendered-hash` varchar(32) NOT NULL DEFAULT '' COMMENT '',
`rendered-html` mediumtext COMMENT 'item.body converted to html',
`object-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams object type',
`object` text COMMENT 'JSON encoded object structure unless it is an implied object (normal post)',
`target-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams target type if applicable (URI)',
`target` text COMMENT 'JSON encoded target structure if used',
`plink` varchar(255) NOT NULL DEFAULT '' COMMENT 'permalink or URL to a displayable copy of the message at its source',
`verb` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams verb',
PRIMARY KEY(`id`),
UNIQUE INDEX `uri-plink-hash` (`uri-plink-hash`),
FULLTEXT INDEX `title-content-warning-body` (`title`,`content-warning`,`body`),
INDEX `uri` (`uri`(191)),
INDEX `plink` (`plink`(191)),
INDEX `uri-id` (`uri-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts';
-- --
-- TABLE locks -- TABLE locks
-- --
@ -1107,6 +1058,33 @@ CREATE TABLE IF NOT EXISTS `post-category` (
FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to categories'; ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to categories';
--
-- TABLE post-content
--
CREATE TABLE IF NOT EXISTS `post-content` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
`content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`body` mediumtext COMMENT 'item body content',
`raw-body` mediumtext COMMENT 'Body without embedded media links',
`location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated',
`coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated',
`language` text COMMENT 'Language information about this post',
`app` varchar(255) NOT NULL DEFAULT '' COMMENT 'application which generated this item',
`rendered-hash` varchar(32) NOT NULL DEFAULT '' COMMENT '',
`rendered-html` mediumtext COMMENT 'item.body converted to html',
`object-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams object type',
`object` text COMMENT 'JSON encoded object structure unless it is an implied object (normal post)',
`target-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams target type if applicable (URI)',
`target` text COMMENT 'JSON encoded target structure if used',
`resource-id` varchar(32) NOT NULL DEFAULT '' COMMENT 'Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type',
`plink` varchar(255) NOT NULL DEFAULT '' COMMENT 'permalink or URL to a displayable copy of the message at its source',
PRIMARY KEY(`uri-id`),
INDEX `plink` (`plink`(191)),
FULLTEXT INDEX `title-content-warning-body` (`title`,`content-warning`,`body`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts';
-- --
-- TABLE post-delivery-data -- TABLE post-delivery-data
-- --
@ -1528,7 +1506,6 @@ CREATE VIEW `post-view` AS SELECT
`item`.`commented` AS `commented`, `item`.`commented` AS `commented`,
`item`.`received` AS `received`, `item`.`received` AS `received`,
`item`.`changed` AS `changed`, `item`.`changed` AS `changed`,
`item`.`resource-id` AS `resource-id`,
`item`.`post-type` AS `post-type`, `item`.`post-type` AS `post-type`,
`item`.`private` AS `private`, `item`.`private` AS `private`,
`item`.`pubmail` AS `pubmail`, `item`.`pubmail` AS `pubmail`,
@ -1546,21 +1523,22 @@ CREATE VIEW `post-view` AS SELECT
`item`.`vid` AS `vid`, `item`.`vid` AS `vid`,
`item`.`psid` AS `psid`, `item`.`psid` AS `psid`,
IF (`item`.`vid` IS NULL, '', `verb`.`name`) AS `verb`, IF (`item`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
`item-content`.`title` AS `title`, `post-content`.`title` AS `title`,
`item-content`.`content-warning` AS `content-warning`, `post-content`.`content-warning` AS `content-warning`,
`item-content`.`raw-body` AS `raw-body`, `post-content`.`raw-body` AS `raw-body`,
`item-content`.`body` AS `body`, `post-content`.`body` AS `body`,
`item-content`.`rendered-hash` AS `rendered-hash`, `post-content`.`rendered-hash` AS `rendered-hash`,
`item-content`.`rendered-html` AS `rendered-html`, `post-content`.`rendered-html` AS `rendered-html`,
`item-content`.`language` AS `language`, `post-content`.`language` AS `language`,
`item-content`.`plink` AS `plink`, `post-content`.`plink` AS `plink`,
`item-content`.`location` AS `location`, `post-content`.`location` AS `location`,
`item-content`.`coord` AS `coord`, `post-content`.`coord` AS `coord`,
`item-content`.`app` AS `app`, `post-content`.`app` AS `app`,
`item-content`.`object-type` AS `object-type`, `post-content`.`object-type` AS `object-type`,
`item-content`.`object` AS `object`, `post-content`.`object` AS `object`,
`item-content`.`target-type` AS `target-type`, `post-content`.`target-type` AS `target-type`,
`item-content`.`target` AS `target`, `post-content`.`target` AS `target`,
`post-content`.`resource-id` AS `resource-id`,
`item`.`contact-id` AS `contact-id`, `item`.`contact-id` AS `contact-id`,
`contact`.`url` AS `contact-link`, `contact`.`url` AS `contact-link`,
`contact`.`addr` AS `contact-addr`, `contact`.`addr` AS `contact-addr`,
@ -1651,7 +1629,7 @@ CREATE VIEW `post-view` AS SELECT
LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid` LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid`
LEFT JOIN `event` ON `event`.`id` = `item`.`event-id` LEFT JOIN `event` ON `event`.`id` = `item`.`event-id`
LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `item`.`uri-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `item`.`uri-id`
LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id` LEFT JOIN `post-content` ON `post-content`.`uri-id` = `item`.`uri-id`
LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `item`.`uri-id` AND `item`.`origin` LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `item`.`uri-id` AND `item`.`origin`
LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid` LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid`
STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`uri-id` = `item`.`parent-uri-id` AND `parent-item`.`uid` = `item`.`uid` STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`uri-id` = `item`.`parent-uri-id` AND `parent-item`.`uid` = `item`.`uid`
@ -1683,7 +1661,6 @@ CREATE VIEW `post-thread-view` AS SELECT
`thread`.`commented` AS `commented`, `thread`.`commented` AS `commented`,
`thread`.`received` AS `received`, `thread`.`received` AS `received`,
`thread`.`changed` AS `changed`, `thread`.`changed` AS `changed`,
`item`.`resource-id` AS `resource-id`,
`thread`.`post-type` AS `post-type`, `thread`.`post-type` AS `post-type`,
`thread`.`private` AS `private`, `thread`.`private` AS `private`,
`thread`.`pubmail` AS `pubmail`, `thread`.`pubmail` AS `pubmail`,
@ -1702,21 +1679,22 @@ CREATE VIEW `post-thread-view` AS SELECT
`item`.`vid` AS `vid`, `item`.`vid` AS `vid`,
`item`.`psid` AS `psid`, `item`.`psid` AS `psid`,
IF (`item`.`vid` IS NULL, '', `verb`.`name`) AS `verb`, IF (`item`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
`item-content`.`title` AS `title`, `post-content`.`title` AS `title`,
`item-content`.`content-warning` AS `content-warning`, `post-content`.`content-warning` AS `content-warning`,
`item-content`.`raw-body` AS `raw-body`, `post-content`.`raw-body` AS `raw-body`,
`item-content`.`body` AS `body`, `post-content`.`body` AS `body`,
`item-content`.`rendered-hash` AS `rendered-hash`, `post-content`.`rendered-hash` AS `rendered-hash`,
`item-content`.`rendered-html` AS `rendered-html`, `post-content`.`rendered-html` AS `rendered-html`,
`item-content`.`language` AS `language`, `post-content`.`language` AS `language`,
`item-content`.`plink` AS `plink`, `post-content`.`plink` AS `plink`,
`item-content`.`location` AS `location`, `post-content`.`location` AS `location`,
`item-content`.`coord` AS `coord`, `post-content`.`coord` AS `coord`,
`item-content`.`app` AS `app`, `post-content`.`app` AS `app`,
`item-content`.`object-type` AS `object-type`, `post-content`.`object-type` AS `object-type`,
`item-content`.`object` AS `object`, `post-content`.`object` AS `object`,
`item-content`.`target-type` AS `target-type`, `post-content`.`target-type` AS `target-type`,
`item-content`.`target` AS `target`, `post-content`.`target` AS `target`,
`post-content`.`resource-id` AS `resource-id`,
`thread`.`contact-id` AS `contact-id`, `thread`.`contact-id` AS `contact-id`,
`contact`.`url` AS `contact-link`, `contact`.`url` AS `contact-link`,
`contact`.`addr` AS `contact-addr`, `contact`.`addr` AS `contact-addr`,
@ -1807,7 +1785,7 @@ CREATE VIEW `post-thread-view` AS SELECT
LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid` LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid`
LEFT JOIN `event` ON `event`.`id` = `item`.`event-id` LEFT JOIN `event` ON `event`.`id` = `item`.`event-id`
LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `thread`.`uri-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `thread`.`uri-id`
LEFT JOIN `item-content` ON `item-content`.`uri-id` = `thread`.`uri-id` LEFT JOIN `post-content` ON `post-content`.`uri-id` = `thread`.`uri-id`
LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `thread`.`uri-id` AND `thread`.`origin` LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `thread`.`uri-id` AND `thread`.`origin`
LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid` LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid`
STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`id` = `item`.`parent` STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`id` = `item`.`parent`

4
include/enotify.php
View file

@ -20,6 +20,7 @@
*/ */
use Friendica\Content\Text\BBCode; use Friendica\Content\Text\BBCode;
use Friendica\Content\Text\Plaintext;
use Friendica\Core\Hook; use Friendica\Core\Hook;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Core\Renderer; use Friendica\Core\Renderer;
@ -28,7 +29,6 @@ use Friendica\Database\DBA;
use Friendica\DI; use Friendica\DI;
use Friendica\Model\Contact; use Friendica\Model\Contact;
use Friendica\Model\Item; use Friendica\Model\Item;
use Friendica\Model\ItemContent;
use Friendica\Model\Notification; use Friendica\Model\Notification;
use Friendica\Model\Post; use Friendica\Model\Post;
use Friendica\Model\User; use Friendica\Model\User;
@ -176,7 +176,7 @@ function notification($params)
$item_post_type = Item::postType($item); $item_post_type = Item::postType($item);
$content = ItemContent::getPlaintextPost($item, 70); $content = Plaintext::getPost($item, 70);
if (!empty($content['text'])) { if (!empty($content['text'])) {
$title = '"' . trim(str_replace("\n", " ", $content['text'])) . '"'; $title = '"' . trim(str_replace("\n", " ", $content['text'])) . '"';
} else { } else {

161
src/Content/Text/Plaintext.php
View file

@ -21,6 +21,9 @@
namespace Friendica\Content\Text; namespace Friendica\Content\Text;
use Friendica\Core\Protocol;
use Friendica\DI;
class Plaintext class Plaintext
{ {
/** /**
@ -88,4 +91,162 @@ class Plaintext
return $res; return $res;
} }
/**
* Convert a message into plaintext for connectors to other networks
*
* @param array $item The message array that is about to be posted
* @param int $limit The maximum number of characters when posting to that network
* @param bool $includedlinks Has an attached link to be included into the message?
* @param int $htmlmode This controls the behavior of the BBCode conversion
* @param string $target_network Name of the network where the post should go to.
*
* @return array Same array structure than \Friendica\Content\Text\BBCode::getAttachedData
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @see \Friendica\Content\Text\BBCode::getAttachedData
*
*/
public static function getPost($item, $limit = 0, $includedlinks = false, $htmlmode = BBCode::API, $target_network = '')
{
// Remove hashtags
$URLSearchString = '^\[\]';
$body = preg_replace("/([#@])\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '$1$3', $item['body']);
// Add an URL element if the text contains a raw link
$body = preg_replace('/([^\]\=\'"]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism',
'$1[url]$2[/url]', $body);
// Remove the abstract
$body = BBCode::stripAbstract($body);
// At first look at data that is attached via "type-..." stuff
// This will hopefully replaced with a dedicated bbcode later
//$post = self::getAttachedData($b['body']);
$post = BBCode::getAttachedData($body, $item);
if (($item['title'] != '') && ($post['text'] != '')) {
$post['text'] = trim($item['title'] . "\n\n" . $post['text']);
} elseif ($item['title'] != '') {
$post['text'] = trim($item['title']);
}
$abstract = '';
// Fetch the abstract from the given target network
if ($target_network != '') {
$default_abstract = BBCode::getAbstract($item['body']);
$abstract = BBCode::getAbstract($item['body'], $target_network);
// If we post to a network with no limit we only fetch
// an abstract exactly for this network
if (($limit == 0) && ($abstract == $default_abstract)) {
$abstract = '';
}
} else {// Try to guess the correct target network
switch ($htmlmode) {
case BBCode::TWITTER:
$abstract = BBCode::getAbstract($item['body'], Protocol::TWITTER);
break;
case BBCode::OSTATUS:
$abstract = BBCode::getAbstract($item['body'], Protocol::STATUSNET);
break;
default: // We don't know the exact target.
// We fetch an abstract since there is a posting limit.
if ($limit > 0) {
$abstract = BBCode::getAbstract($item['body']);
}
}
}
if ($abstract != '') {
$post['text'] = $abstract;
if ($post['type'] == 'text') {
$post['type'] = 'link';
$post['url'] = $item['plink'];
}
}
$html = BBCode::convert($post['text'] . ($post['after'] ?? ''), false, $htmlmode);
$msg = HTML::toPlaintext($html, 0, true);
$msg = trim(html_entity_decode($msg, ENT_QUOTES, 'UTF-8'));
$link = '';
if ($includedlinks) {
if ($post['type'] == 'link') {
$link = $post['url'];
} elseif ($post['type'] == 'text') {
$link = $post['url'] ?? '';
} elseif ($post['type'] == 'video') {
$link = $post['url'];
} elseif ($post['type'] == 'photo') {
$link = $post['image'];
}
if (($msg == '') && isset($post['title'])) {
$msg = trim($post['title']);
}
if (($msg == '') && isset($post['description'])) {
$msg = trim($post['description']);
}
// If the link is already contained in the post, then it neeedn't to be added again
// But: if the link is beyond the limit, then it has to be added.
if (($link != '') && strstr($msg, $link)) {
$pos = strpos($msg, $link);
// Will the text be shortened in the link?
// Or is the link the last item in the post?
if (($limit > 0) && ($pos < $limit) && (($pos + 23 > $limit) || ($pos + strlen($link) == strlen($msg)))) {
$msg = trim(str_replace($link, '', $msg));
} elseif (($limit == 0) || ($pos < $limit)) {
// The limit has to be increased since it will be shortened - but not now
// Only do it with Twitter
if (($limit > 0) && (strlen($link) > 23) && ($htmlmode == BBCode::TWITTER)) {
$limit = $limit - 23 + strlen($link);
}
$link = '';
if ($post['type'] == 'text') {
unset($post['url']);
}
}
}
}
if ($limit > 0) {
// Reduce multiple spaces
// When posted to a network with limited space, we try to gain space where possible
while (strpos($msg, ' ') !== false) {
$msg = str_replace(' ', ' ', $msg);
}
// Twitter is using its own limiter, so we always assume that shortened links will have this length
if (iconv_strlen($link, 'UTF-8') > 0) {
$limit = $limit - 23;
}
if (iconv_strlen($msg, 'UTF-8') > $limit) {
if (($post['type'] == 'text') && isset($post['url'])) {
$post['url'] = $item['plink'];
} elseif (!isset($post['url'])) {
$limit = $limit - 23;
$post['url'] = $item['plink'];
} elseif (strpos($item['body'], '[share') !== false) {
$post['url'] = $item['plink'];
} elseif (DI::pConfig()->get($item['uid'], 'system', 'no_intelligent_shortening')) {
$post['url'] = $item['plink'];
}
$msg = self::shorten($msg, $limit);
}
}
$post['text'] = trim($msg);
return $post;
}
} }

4
src/Core/Update.php
View file

@ -72,8 +72,8 @@ class Update
} }
} }
// The postupdate has to completed version 1281 for the new post views to take over // The postupdate has to completed version 1288 for the new post views to take over
$postupdate = DI::config()->get("system", "post_update_version"); $postupdate = DI::config()->get("system", "post_update_version", NEW_TABLE_STRUCTURE_VERSION);
if ($postupdate < NEW_TABLE_STRUCTURE_VERSION) { if ($postupdate < NEW_TABLE_STRUCTURE_VERSION) {
$error = DI::l10n()->t('Updates from postupdate version %s are not supported. Please update at least to version 2021.01 and wait until the postupdate finished version 1383.', $postupdate); $error = DI::l10n()->t('Updates from postupdate version %s are not supported. Please update at least to version 2021.01 and wait until the postupdate finished version 1383.', $postupdate);
if (DI::mode()->getExecutor() == Mode::INDEX) { if (DI::mode()->getExecutor() == Mode::INDEX) {

8
src/Database/DBStructure.php
View file

@ -72,8 +72,14 @@ class DBStructure
*/ */
public static function dropTables(bool $execute) public static function dropTables(bool $execute)
{ {
$postupdate = DI::config()->get("system", "post_update_version", PostUpdate::VERSION);
if ($postupdate < PostUpdate::VERSION) {
echo DI::l10n()->t('The post update is at version %d, it has to be at %d to safely drop the tables.', $postupdate, PostUpdate::VERSION);
return;
}
$old_tables = ['fserver', 'gcign', 'gcontact', 'gcontact-relation', 'gfollower' ,'glink', 'item-delivery-data', $old_tables = ['fserver', 'gcign', 'gcontact', 'gcontact-relation', 'gfollower' ,'glink', 'item-delivery-data',
'item_id', 'poll', 'poll_result', 'queue', 'retriever_rule', 'sign', 'spam', 'term']; 'item-activity', 'item-content', 'item_id', 'poll', 'poll_result', 'queue', 'retriever_rule', 'sign', 'spam', 'term'];
$tables = DBA::selectToArray(['INFORMATION_SCHEMA' => 'TABLES'], ['TABLE_NAME'], $tables = DBA::selectToArray(['INFORMATION_SCHEMA' => 'TABLES'], ['TABLE_NAME'],
['TABLE_SCHEMA' => DBA::databaseName(), 'TABLE_TYPE' => 'BASE TABLE']); ['TABLE_SCHEMA' => DBA::databaseName(), 'TABLE_TYPE' => 'BASE TABLE']);

12
src/Database/PostUpdate.php
View file

@ -230,6 +230,11 @@ class PostUpdate
return true; return true;
} }
if (!DBStructure::existsTable('item-content')) {
DI::config()->set('system', 'post_update_version', 1342);
return true;
}
$id = DI::config()->get('system', 'post_update_version_1341_id', 0); $id = DI::config()->get('system', 'post_update_version_1341_id', 0);
Logger::info('Start', ['item' => $id]); Logger::info('Start', ['item' => $id]);
@ -283,7 +288,7 @@ class PostUpdate
return true; return true;
} }
if (!DBStructure::existsTable('term')) { if (!DBStructure::existsTable('term') || !DBStructure::existsTable('item-content')) {
DI::config()->set('system', 'post_update_version', 1342); DI::config()->set('system', 'post_update_version', 1342);
return true; return true;
} }
@ -510,6 +515,11 @@ class PostUpdate
return true; return true;
} }
if (!DBStructure::existsTable('item-activity')) {
DI::config()->set('system', 'post_update_version', 1347);
return true;
}
$id = DI::config()->get("system", "post_update_version_1347_id", 0); $id = DI::config()->get("system", "post_update_version_1347_id", 0);
Logger::info('Start', ['item' => $id]); Logger::info('Start', ['item' => $id]);

75
src/Model/Item.php
View file

@ -99,12 +99,12 @@ class Item
'author-id', 'author-link', 'owner-link', 'contact-uid', 'author-id', 'author-link', 'owner-link', 'contact-uid',
'signed_text', 'network']; 'signed_text', 'network'];
// Field list for "item-content" table that is mixed with the item table // Field list for "post-content" table that is mixed with the item table
const MIXED_CONTENT_FIELDLIST = ['title', 'content-warning', 'body', 'location', const MIXED_CONTENT_FIELDLIST = ['title', 'content-warning', 'body', 'location',
'coord', 'app', 'rendered-hash', 'rendered-html', 'verb', 'coord', 'app', 'rendered-hash', 'rendered-html', 'verb',
'object-type', 'object', 'target-type', 'target', 'plink']; 'object-type', 'object', 'target-type', 'target', 'plink'];
// Field list for "item-content" table that is not present in the "item" table // Field list for "post-content" table that is not present in the "item" table
const CONTENT_FIELDLIST = ['language', 'raw-body']; const CONTENT_FIELDLIST = ['language', 'raw-body'];
// All fields in the item table // All fields in the item table
@ -133,7 +133,7 @@ class Item
const PRIVATE = 1; const PRIVATE = 1;
const UNLISTED = 2; const UNLISTED = 2;
const TABLES = ['item', 'user-item', 'item-content', 'post-delivery-data', 'diaspora-interaction']; const TABLES = ['item', 'user-item', 'post-content', 'post-delivery-data', 'diaspora-interaction'];
private static function getItemFields() private static function getItemFields()
{ {
@ -224,7 +224,6 @@ class Item
// Remove all media attachments from the body and store them in the post-media table // Remove all media attachments from the body and store them in the post-media table
$content_fields['raw-body'] = Post\Media::insertFromBody($item['uri-id'], $content_fields['raw-body']); $content_fields['raw-body'] = Post\Media::insertFromBody($item['uri-id'], $content_fields['raw-body']);
$content_fields['raw-body'] = self::setHashtags($content_fields['raw-body']); $content_fields['raw-body'] = self::setHashtags($content_fields['raw-body']);
self::updateContent($content_fields, ['uri-id' => $item['uri-id']]);
} }
if (!empty($fields['file'])) { if (!empty($fields['file'])) {
@ -1054,9 +1053,9 @@ class Item
$notify_type = Delivery::POST; $notify_type = Delivery::POST;
} }
if (!in_array($item['verb'], self::ACTIVITIES) && !self::insertContent($item)) { if (!in_array($item['verb'], self::ACTIVITIES) && !Post\Content::insert($item['uri-id'], $item)) {
// This shouldn't happen // This shouldn't happen
Logger::warning('No content stored, quitting', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'causer-id' => ($item['causer-id'] ?? 0), 'post-type' => $item['post-type'], 'network' => $item['network']]); Logger::warning('No post-content stored, quitting', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'causer-id' => ($item['causer-id'] ?? 0), 'post-type' => $item['post-type'], 'network' => $item['network']]);
return 0; return 0;
} }
@ -1109,9 +1108,6 @@ class Item
} }
} }
// We syncronize the id value of the of the post-user table with the item table
$item['id'] = $id;
$condition = ['uri-id' => $item['uri-id'], 'uid' => $item['uid'], 'network' => $item['network']]; $condition = ['uri-id' => $item['uri-id'], 'uid' => $item['uid'], 'network' => $item['network']];
if (Post::exists($condition)) { if (Post::exists($condition)) {
Logger::notice('Item is already inserted - aborting', $condition); Logger::notice('Item is already inserted - aborting', $condition);
@ -1300,67 +1296,6 @@ class Item
} }
} }
/**
* Insert a new item content entry
*
* @param array $item The item fields that are to be inserted
* @return bool "true" if content was inserted or already existed
* @throws \Exception
*/
private static function insertContent(array $item)
{
$fields = ['uri-plink-hash' => (string)$item['uri-id'], 'uri-id' => $item['uri-id']];
foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
if (isset($item[$field])) {
$fields[$field] = $item[$field];
}
}
$found = DBA::exists('item-content', ['uri-id' => $item['uri-id']]);
if ($found) {
Logger::info('Existing content found', ['uri-id' => $item['uri-id'], 'uri' => $item['uri']]);
return true;
}
DBA::insert('item-content', $fields, Database::INSERT_IGNORE);
$found = DBA::exists('item-content', ['uri-id' => $item['uri-id']]);
if ($found) {
Logger::notice('Content inserted', ['uri-id' => $item['uri-id'], 'uri' => $item['uri']]);
return true;
}
// This shouldn't happen.
Logger::error("Content wasn't inserted", $item);
return false;
}
/**
* Update existing item content entries
*
* @param array $item The item fields that are to be changed
* @param array $condition The condition for finding the item content entries
* @throws \Exception
*/
private static function updateContent($item, $condition)
{
// We have to select only the fields from the "item-content" table
$fields = [];
foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
if (isset($item[$field])) {
$fields[$field] = $item[$field];
}
}
if (empty($fields)) {
return;
}
DBA::update('item-content', $fields, $condition, true);
Logger::info('Updated content', ['condition' => $condition]);
}
/** /**
* Distributes public items to the receivers * Distributes public items to the receivers
* *

236
src/Model/ItemContent.php
View file

@ -1,236 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Model;
use Friendica\Content\Text;
use Friendica\Content\Text\BBCode;
use Friendica\Core\Protocol;
use Friendica\Database\DBA;
use Friendica\DI;
class ItemContent
{
/**
* Search posts for given content
*
* @param string $search
* @param integer $uid
* @param integer $start
* @param integer $limit
* @param integer $last_uriid
* @return array
*/
public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100, int $last_uriid = 0)
{
$condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
AND (NOT `private` OR (`private` AND `uid` = ?))
AND `uri-id` IN (SELECT `uri-id` FROM `post-view` WHERE `network` IN (?, ?, ?, ?))",
$search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS];
if (!empty($last_uriid)) {
$condition = DBA::mergeConditions($condition, ["`uri-id` < ?", $last_uriid]);
}
$params = [
'order' => ['uri-id' => true],
'group_by' => ['uri-id'],
'limit' => [$start, $limit]
];
$tags = Post::select(['uri-id'], $condition, $params);
$uriids = [];
while ($tag = DBA::fetch($tags)) {
$uriids[] = $tag['uri-id'];
}
DBA::close($tags);
return $uriids;
}
public static function countBySearch(string $search, int $uid = 0)
{
$condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
AND (NOT `private` OR (`private` AND `uid` = ?))
AND `uri-id` IN (SELECT `uri-id` FROM `post-view` WHERE `network` IN (?, ?, ?, ?))",
$search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS];
return Post::count($condition);
}
/**
* Convert a message into plaintext for connectors to other networks
*
* @param array $item The message array that is about to be posted
* @param int $limit The maximum number of characters when posting to that network
* @param bool $includedlinks Has an attached link to be included into the message?
* @param int $htmlmode This controls the behavior of the BBCode conversion
* @param string $target_network Name of the network where the post should go to.
*
* @return array Same array structure than \Friendica\Content\Text\BBCode::getAttachedData
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @see \Friendica\Content\Text\BBCode::getAttachedData
*
*/
public static function getPlaintextPost($item, $limit = 0, $includedlinks = false, $htmlmode = BBCode::API, $target_network = '')
{
// Remove hashtags
$URLSearchString = '^\[\]';
$body = preg_replace("/([#@])\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '$1$3', $item['body']);
// Add an URL element if the text contains a raw link
$body = preg_replace('/([^\]\=\'"]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism',
'$1[url]$2[/url]', $body);
// Remove the abstract
$body = Text\BBCode::stripAbstract($body);
// At first look at data that is attached via "type-..." stuff
// This will hopefully replaced with a dedicated bbcode later
//$post = self::getAttachedData($b['body']);
$post = Text\BBCode::getAttachedData($body, $item);
if (($item['title'] != '') && ($post['text'] != '')) {
$post['text'] = trim($item['title'] . "\n\n" . $post['text']);
} elseif ($item['title'] != '') {
$post['text'] = trim($item['title']);
}
$abstract = '';
// Fetch the abstract from the given target network
if ($target_network != '') {
$default_abstract = Text\BBCode::getAbstract($item['body']);
$abstract = Text\BBCode::getAbstract($item['body'], $target_network);
// If we post to a network with no limit we only fetch
// an abstract exactly for this network
if (($limit == 0) && ($abstract == $default_abstract)) {
$abstract = '';
}
} else {// Try to guess the correct target network
switch ($htmlmode) {
case BBCode::TWITTER:
$abstract = Text\BBCode::getAbstract($item['body'], Protocol::TWITTER);
break;
case BBCode::OSTATUS:
$abstract = Text\BBCode::getAbstract($item['body'], Protocol::STATUSNET);
break;
default: // We don't know the exact target.
// We fetch an abstract since there is a posting limit.
if ($limit > 0) {
$abstract = Text\BBCode::getAbstract($item['body']);
}
}
}
if ($abstract != '') {
$post['text'] = $abstract;
if ($post['type'] == 'text') {
$post['type'] = 'link';
$post['url'] = $item['plink'];
}
}
$html = Text\BBCode::convert($post['text'] . ($post['after'] ?? ''), false, $htmlmode);
$msg = Text\HTML::toPlaintext($html, 0, true);
$msg = trim(html_entity_decode($msg, ENT_QUOTES, 'UTF-8'));
$link = '';
if ($includedlinks) {
if ($post['type'] == 'link') {
$link = $post['url'];
} elseif ($post['type'] == 'text') {
$link = $post['url'] ?? '';
} elseif ($post['type'] == 'video') {
$link = $post['url'];
} elseif ($post['type'] == 'photo') {
$link = $post['image'];
}
if (($msg == '') && isset($post['title'])) {
$msg = trim($post['title']);
}
if (($msg == '') && isset($post['description'])) {
$msg = trim($post['description']);
}
// If the link is already contained in the post, then it neeedn't to be added again
// But: if the link is beyond the limit, then it has to be added.
if (($link != '') && strstr($msg, $link)) {
$pos = strpos($msg, $link);
// Will the text be shortened in the link?
// Or is the link the last item in the post?
if (($limit > 0) && ($pos < $limit) && (($pos + 23 > $limit) || ($pos + strlen($link) == strlen($msg)))) {
$msg = trim(str_replace($link, '', $msg));
} elseif (($limit == 0) || ($pos < $limit)) {
// The limit has to be increased since it will be shortened - but not now
// Only do it with Twitter
if (($limit > 0) && (strlen($link) > 23) && ($htmlmode == BBCode::TWITTER)) {
$limit = $limit - 23 + strlen($link);
}
$link = '';
if ($post['type'] == 'text') {
unset($post['url']);
}
}
}
}
if ($limit > 0) {
// Reduce multiple spaces
// When posted to a network with limited space, we try to gain space where possible
while (strpos($msg, ' ') !== false) {
$msg = str_replace(' ', ' ', $msg);
}
// Twitter is using its own limiter, so we always assume that shortened links will have this length
if (iconv_strlen($link, 'UTF-8') > 0) {
$limit = $limit - 23;
}
if (iconv_strlen($msg, 'UTF-8') > $limit) {
if (($post['type'] == 'text') && isset($post['url'])) {
$post['url'] = $item['plink'];
} elseif (!isset($post['url'])) {
$limit = $limit - 23;
$post['url'] = $item['plink'];
} elseif (strpos($item['body'], '[share') !== false) {
$post['url'] = $item['plink'];
} elseif (DI::pConfig()->get($item['uid'], 'system', 'no_intelligent_shortening')) {
$post['url'] = $item['plink'];
}
$msg = Text\Plaintext::shorten($msg, $limit);
}
}
$post['text'] = trim($msg);
return $post;
}
}

6
src/Model/Post.php
View file

@ -446,13 +446,13 @@ class Post
$affected = DBA::affectedRows(); $affected = DBA::affectedRows();
} }
$update_fields = DBStructure::getFieldsForTable('item-content', $fields); $update_fields = DBStructure::getFieldsForTable('post-content', $fields);
if (!empty($update_fields)) { if (!empty($update_fields)) {
$rows = DBA::selectToArray('post-view', ['uri-id'], $condition, ['group_by' => ['uri-id']]); $rows = DBA::selectToArray('post-view', ['uri-id'], $condition, ['group_by' => ['uri-id']]);
$uriids = array_column($rows, 'uri-id'); $uriids = array_column($rows, 'uri-id');
if (!DBA::update('item-content', $update_fields, ['uri-id' => $uriids])) { if (!DBA::update('post-content', $update_fields, ['uri-id' => $uriids])) {
DBA::rollback(); DBA::rollback();
Logger::notice('Updating item-content failed', ['fields' => $update_fields, 'condition' => $condition]); Logger::notice('Updating post-content failed', ['fields' => $update_fields, 'condition' => $condition]);
return false; return false;
} }
$affected = max($affected, DBA::affectedRows()); $affected = max($affected, DBA::affectedRows());

147
src/Model/Post/Content.php Normal file
View file

@ -0,0 +1,147 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Model\Post;
use \BadMethodCallException;
use Friendica\Core\Protocol;
use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\Database\DBStructure;
use Friendica\Model\Post;
class Content
{
/**
* Insert a new post-content entry
*
* @param integer $uri_id
* @param array $fields
* @return bool success
* @throws \Exception
*/
public static function insert(int $uri_id, array $data = [])
{
if (empty($uri_id)) {
throw new BadMethodCallException('Empty URI_id');
}
if (DBA::exists('post-content', ['uri-id' => $uri_id])) {
return false;
}
$fields = DBStructure::getFieldsForTable('post-content', $data);
// Additionally assign the key fields
$fields['uri-id'] = $uri_id;
return DBA::insert('post-content', $fields, Database::INSERT_IGNORE);
}
/**
* Update a post content entry
*
* @param integer $uri_id
* @param array $data
* @param bool $insert_if_missing
* @return bool
* @throws \Exception
*/
public static function update(int $uri_id, array $data = [], bool $insert_if_missing = false)
{
if (empty($uri_id)) {
throw new BadMethodCallException('Empty URI_id');
}
$fields = DBStructure::getFieldsForTable('post-content', $data);
// Remove the key fields
unset($fields['uri-id']);
if (empty($fields)) {
return true;
}
return DBA::update('post-content', $fields, ['uri-id' => $uri_id], $insert_if_missing ? true : []);
}
/**
* Delete a row from the post-content table
*
* @param array $conditions Field condition(s)
* @param array $options
* - cascade: If true we delete records in other tables that depend on the one we're deleting through
* relations (default: true)
*
* @return boolean was the delete successful?
* @throws \Exception
*/
public static function delete(array $conditions, array $options = [])
{
return DBA::delete('post-content', $conditions, $options);
}
/**
* Search posts for given content
*
* @param string $search
* @param integer $uid
* @param integer $start
* @param integer $limit
* @param integer $last_uriid
* @return array
*/
public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100, int $last_uriid = 0)
{
$condition = ["`uri-id` IN (SELECT `uri-id` FROM `post-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
AND (NOT `private` OR (`private` AND `uid` = ?)) AND (`network` IN (?, ?, ?, ?) OR (`uid` = ? AND `uid` != ?))",
$search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, $uid, 0];
if (!empty($last_uriid)) {
$condition = DBA::mergeConditions($condition, ["`uri-id` < ?", $last_uriid]);
}
$params = [
'order' => ['uri-id' => true],
'group_by' => ['uri-id'],
'limit' => [$start, $limit]
];
$tags = Post::select(['uri-id'], $condition, $params);
$uriids = [];
while ($tag = DBA::fetch($tags)) {
$uriids[] = $tag['uri-id'];
}
DBA::close($tags);
return $uriids;
}
public static function countBySearch(string $search, int $uid = 0)
{
$condition = ["`uri-id` IN (SELECT `uri-id` FROM `post-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
AND (NOT `private` OR (`private` AND `uid` = ?)) AND (`network` IN (?, ?, ?, ?) OR (`uid` = ? AND `uid` != ?))",
$search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, $uid, 0];
return Post::count($condition);
}
}

4
src/Module/Search/Index.php
View file

@ -166,8 +166,8 @@ class Index extends BaseSearch
$count = Tag::countByTag($search, local_user()); $count = Tag::countByTag($search, local_user());
} else { } else {
Logger::info('Start fulltext search.', ['q' => $search]); Logger::info('Start fulltext search.', ['q' => $search]);
$uriids = ItemContent::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage(), $last_uriid); $uriids = Post\Content::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage(), $last_uriid);
$count = ItemContent::countBySearch($search, local_user()); $count = Post\Content::countBySearch($search, local_user());
} }
if (!empty($uriids)) { if (!empty($uriids)) {

4
src/Worker/Expire.php
View file

@ -55,8 +55,8 @@ class Expire
// Normally we shouldn't have orphaned data at all. // Normally we shouldn't have orphaned data at all.
// If we do have some, then we have to check why. // If we do have some, then we have to check why.
Logger::log('Deleting orphaned item content - start', Logger::DEBUG); Logger::log('Deleting orphaned item content - start', Logger::DEBUG);
$condition = ["NOT EXISTS (SELECT `uri-id` FROM `item` WHERE `item`.`uri-id` = `item-content`.`uri-id`)"]; $condition = ["NOT EXISTS (SELECT `uri-id` FROM `item` WHERE `item`.`uri-id` = `post-content`.`uri-id`)"];
DBA::delete('item-content', $condition); DBA::delete('post-content', $condition);
Logger::log('Orphaned item content deleted: ' . DBA::affectedRows(), Logger::DEBUG); Logger::log('Orphaned item content deleted: ' . DBA::affectedRows(), Logger::DEBUG);
// make this optional as it could have a performance impact on large sites // make this optional as it could have a performance impact on large sites

78
static/dbstructure.config.php
View file

@ -55,7 +55,7 @@
use Friendica\Database\DBA; use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) { if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1395); define('DB_UPDATE_VERSION', 1396);
} }
return [ return [
@ -858,55 +858,6 @@ return [
"causer-id" => ["causer-id"], "causer-id" => ["causer-id"],
] ]
], ],
"item-activity" => [
"comment" => "Activities for items",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"],
"uri" => ["type" => "varchar(255)", "comment" => ""],
"uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"uri-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"],
"activity" => ["type" => "smallint unsigned", "not null" => "1", "default" => "0", "comment" => ""]
],
"indexes" => [
"PRIMARY" => ["id"],
"uri-hash" => ["UNIQUE", "uri-hash"],
"uri" => ["uri(191)"],
"uri-id" => ["uri-id"]
]
],
"item-content" => [
"comment" => "Content for all posts",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"],
"uri" => ["type" => "varchar(255)", "comment" => ""],
"uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"uri-plink-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"],
"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"body" => ["type" => "mediumtext", "comment" => "item body content"],
"raw-body" => ["type" => "mediumtext", "comment" => "Body without embedded media links"],
"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
"coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
"language" => ["type" => "text", "comment" => "Language information about this post"],
"app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"],
"rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
"rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"],
"object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"],
"object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"],
"target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"],
"target" => ["type" => "text", "comment" => "JSON encoded target structure if used"],
"plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"],
"verb" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams verb"]
],
"indexes" => [
"PRIMARY" => ["id"],
"uri-plink-hash" => ["UNIQUE", "uri-plink-hash"],
"title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"],
"uri" => ["uri(191)"],
"plink" => ["plink(191)"],
"uri-id" => ["uri-id"]
]
],
"locks" => [ "locks" => [
"comment" => "", "comment" => "",
"fields" => [ "fields" => [
@ -1161,6 +1112,33 @@ return [
"uid" => ["uid"], "uid" => ["uid"],
] ]
], ],
"post-content" => [
"comment" => "Content for all posts",
"fields" => [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"body" => ["type" => "mediumtext", "comment" => "item body content"],
"raw-body" => ["type" => "mediumtext", "comment" => "Body without embedded media links"],
"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
"coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
"language" => ["type" => "text", "comment" => "Language information about this post"],
"app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"],
"rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
"rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"],
"object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"],
"object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"],
"target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"],
"target" => ["type" => "text", "comment" => "JSON encoded target structure if used"],
"resource-id" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type"],
"plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"]
],
"indexes" => [
"PRIMARY" => ["uri-id"],
"plink" => ["plink(191)"],
"title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"],
]
],
"post-delivery-data" => [ "post-delivery-data" => [
"comment" => "Delivery data for items", "comment" => "Delivery data for items",
"fields" => [ "fields" => [

68
static/dbview.config.php
View file

@ -60,7 +60,6 @@
"commented" => ["item", "commented"], "commented" => ["item", "commented"],
"received" => ["item", "received"], "received" => ["item", "received"],
"changed" => ["item", "changed"], "changed" => ["item", "changed"],
"resource-id" => ["item", "resource-id"],
"post-type" => ["item", "post-type"], "post-type" => ["item", "post-type"],
"private" => ["item", "private"], "private" => ["item", "private"],
"pubmail" => ["item", "pubmail"], "pubmail" => ["item", "pubmail"],
@ -78,21 +77,22 @@
"vid" => ["item", "vid"], "vid" => ["item", "vid"],
"psid" => ["item", "psid"], "psid" => ["item", "psid"],
"verb" => "IF (`item`.`vid` IS NULL, '', `verb`.`name`)", "verb" => "IF (`item`.`vid` IS NULL, '', `verb`.`name`)",
"title" => ["item-content", "title"], "title" => ["post-content", "title"],
"content-warning" => ["item-content", "content-warning"], "content-warning" => ["post-content", "content-warning"],
"raw-body" => ["item-content", "raw-body"], "raw-body" => ["post-content", "raw-body"],
"body" => ["item-content", "body"], "body" => ["post-content", "body"],
"rendered-hash" => ["item-content", "rendered-hash"], "rendered-hash" => ["post-content", "rendered-hash"],
"rendered-html" => ["item-content", "rendered-html"], "rendered-html" => ["post-content", "rendered-html"],
"language" => ["item-content", "language"], "language" => ["post-content", "language"],
"plink" => ["item-content", "plink"], "plink" => ["post-content", "plink"],
"location" => ["item-content", "location"], "location" => ["post-content", "location"],
"coord" => ["item-content", "coord"], "coord" => ["post-content", "coord"],
"app" => ["item-content", "app"], "app" => ["post-content", "app"],
"object-type" => ["item-content", "object-type"], "object-type" => ["post-content", "object-type"],
"object" => ["item-content", "object"], "object" => ["post-content", "object"],
"target-type" => ["item-content", "target-type"], "target-type" => ["post-content", "target-type"],
"target" => ["item-content", "target"], "target" => ["post-content", "target"],
"resource-id" => ["post-content", "resource-id"],
"contact-id" => ["item", "contact-id"], "contact-id" => ["item", "contact-id"],
"contact-link" => ["contact", "url"], "contact-link" => ["contact", "url"],
"contact-addr" => ["contact", "addr"], "contact-addr" => ["contact", "addr"],
@ -184,7 +184,7 @@
LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid` LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid`
LEFT JOIN `event` ON `event`.`id` = `item`.`event-id` LEFT JOIN `event` ON `event`.`id` = `item`.`event-id`
LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `item`.`uri-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `item`.`uri-id`
LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id` LEFT JOIN `post-content` ON `post-content`.`uri-id` = `item`.`uri-id`
LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `item`.`uri-id` AND `item`.`origin` LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `item`.`uri-id` AND `item`.`origin`
LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid` LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid`
STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`uri-id` = `item`.`parent-uri-id` AND `parent-item`.`uid` = `item`.`uid` STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`uri-id` = `item`.`parent-uri-id` AND `parent-item`.`uid` = `item`.`uid`
@ -213,7 +213,6 @@
"commented" => ["thread", "commented"], "commented" => ["thread", "commented"],
"received" => ["thread", "received"], "received" => ["thread", "received"],
"changed" => ["thread", "changed"], "changed" => ["thread", "changed"],
"resource-id" => ["item", "resource-id"],
"post-type" => ["thread", "post-type"], "post-type" => ["thread", "post-type"],
"private" => ["thread", "private"], "private" => ["thread", "private"],
"pubmail" => ["thread", "pubmail"], "pubmail" => ["thread", "pubmail"],
@ -232,21 +231,22 @@
"vid" => ["item", "vid"], "vid" => ["item", "vid"],
"psid" => ["item", "psid"], "psid" => ["item", "psid"],
"verb" => "IF (`item`.`vid` IS NULL, '', `verb`.`name`)", "verb" => "IF (`item`.`vid` IS NULL, '', `verb`.`name`)",
"title" => ["item-content", "title"], "title" => ["post-content", "title"],
"content-warning" => ["item-content", "content-warning"], "content-warning" => ["post-content", "content-warning"],
"raw-body" => ["item-content", "raw-body"], "raw-body" => ["post-content", "raw-body"],
"body" => ["item-content", "body"], "body" => ["post-content", "body"],
"rendered-hash" => ["item-content", "rendered-hash"], "rendered-hash" => ["post-content", "rendered-hash"],
"rendered-html" => ["item-content", "rendered-html"], "rendered-html" => ["post-content", "rendered-html"],
"language" => ["item-content", "language"], "language" => ["post-content", "language"],
"plink" => ["item-content", "plink"], "plink" => ["post-content", "plink"],
"location" => ["item-content", "location"], "location" => ["post-content", "location"],
"coord" => ["item-content", "coord"], "coord" => ["post-content", "coord"],
"app" => ["item-content", "app"], "app" => ["post-content", "app"],
"object-type" => ["item-content", "object-type"], "object-type" => ["post-content", "object-type"],
"object" => ["item-content", "object"], "object" => ["post-content", "object"],
"target-type" => ["item-content", "target-type"], "target-type" => ["post-content", "target-type"],
"target" => ["item-content", "target"], "target" => ["post-content", "target"],
"resource-id" => ["post-content", "resource-id"],
"contact-id" => ["thread", "contact-id"], "contact-id" => ["thread", "contact-id"],
"contact-link" => ["contact", "url"], "contact-link" => ["contact", "url"],
"contact-addr" => ["contact", "addr"], "contact-addr" => ["contact", "addr"],
@ -338,7 +338,7 @@
LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid` LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid`
LEFT JOIN `event` ON `event`.`id` = `item`.`event-id` LEFT JOIN `event` ON `event`.`id` = `item`.`event-id`
LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `thread`.`uri-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `thread`.`uri-id`
LEFT JOIN `item-content` ON `item-content`.`uri-id` = `thread`.`uri-id` LEFT JOIN `post-content` ON `post-content`.`uri-id` = `thread`.`uri-id`
LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `thread`.`uri-id` AND `thread`.`origin` LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `thread`.`uri-id` AND `thread`.`origin`
LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid` LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid`
STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`id` = `item`.`parent` STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`id` = `item`.`parent`

14
tests/datasets/api.fixture.php
View file

@ -179,46 +179,34 @@ return [
'guid' => '6', 'guid' => '6',
], ],
], ],
'item-content' => [ 'post-content' => [
[ [
'id' => 1,
'uri-id' => 1, 'uri-id' => 1,
'uri-plink-hash' => '1',
'body' => 'Parent status', 'body' => 'Parent status',
'plink' => 'http://localhost/display/1', 'plink' => 'http://localhost/display/1',
], ],
[ [
'id' => 2,
'uri-id' => 2, 'uri-id' => 2,
'uri-plink-hash' => '2',
'body' => 'Reply', 'body' => 'Reply',
'plink' => 'http://localhost/display/2', 'plink' => 'http://localhost/display/2',
], ],
[ [
'id' => 3,
'uri-id' => 3, 'uri-id' => 3,
'uri-plink-hash' => '3',
'body' => 'Other user status', 'body' => 'Other user status',
'plink' => 'http://localhost/display/3', 'plink' => 'http://localhost/display/3',
], ],
[ [
'id' => 4,
'uri-id' => 4, 'uri-id' => 4,
'uri-plink-hash' => '4',
'body' => 'Friend user reply', 'body' => 'Friend user reply',
'plink' => 'http://localhost/display/4', 'plink' => 'http://localhost/display/4',
], ],
[ [
'id' => 5,
'uri-id' => 5, 'uri-id' => 5,
'uri-plink-hash' => '5',
'body' => '[share]Shared status[/share]', 'body' => '[share]Shared status[/share]',
'plink' => 'http://localhost/display/5', 'plink' => 'http://localhost/display/5',
], ],
[ [
'id' => 6,
'uri-id' => 6, 'uri-id' => 6,
'uri-plink-hash' => '6',
'body' => 'Friend user status', 'body' => 'Friend user status',
'plink' => 'http://localhost/display/6', 'plink' => 'http://localhost/display/6',
], ],

24
update.php
View file

@ -266,6 +266,10 @@ function update_1348()
function update_1349() function update_1349()
{ {
if (!DBStructure::existsTable('item-activity')) {
return Update::SUCCESS;
}
$correct = true; $correct = true;
foreach (Item::ACTIVITIES as $index => $activity) { foreach (Item::ACTIVITIES as $index => $activity) {
if (!DBA::exists('verb', ['id' => $index + 1, 'name' => $activity])) { if (!DBA::exists('verb', ['id' => $index + 1, 'name' => $activity])) {
@ -690,3 +694,23 @@ function update_1395()
} }
return Update::SUCCESS; return Update::SUCCESS;
} }
function update_1396()
{
if (!DBStructure::existsTable('item-content')) {
return Update::SUCCESS;
}
if (!DBA::e("INSERT IGNORE INTO `post-content`(`uri-id`, `title`, `content-warning`, `body`, `raw-body`,
`location`, `coord`, `language`, `app`, `rendered-hash`, `rendered-html`,
`object-type`, `object`, `target-type`, `target`, `resource-id`, `plink`)
SELECT `item-content`.`uri-id`, `item-content`.`title`, `item-content`.`content-warning`,
`item-content`.`body`, `item-content`.`raw-body`, `item-content`.`location`, `item-content`.`coord`,
`item-content`.`language`, `item-content`.`app`, `item-content`.`rendered-hash`,
`item-content`.`rendered-html`, `item-content`.`object-type`, `item-content`.`object`,
`item-content`.`target-type`, `item-content`.`target`, `item`.`resource-id`, `item-content`.`plink`
FROM `item-content` INNER JOIN `item` ON `item`.`uri-id` = `item-content`.`uri-id`")) {
return Update::FAILED;
}
return Update::SUCCESS;
}