From c3398511b45a89f8a11626d88dc6bd27bc2494e9 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 30 Jan 2021 22:03:53 +0000 Subject: [PATCH] New table "post-content" --- include/enotify.php | 4 +- src/Content/Text/Plaintext.php | 161 ++++++++++++++++++++++ src/Core/Update.php | 4 +- src/Database/DBStructure.php | 8 +- src/Database/PostUpdate.php | 12 +- src/Model/Item.php | 72 +--------- src/Model/ItemContent.php | 236 --------------------------------- src/Model/Post.php | 6 +- src/Model/Post/Content.php | 147 ++++++++++++++++++++ src/Module/Search/Index.php | 4 +- src/Worker/Expire.php | 4 +- static/dbstructure.config.php | 78 ++++------- static/dbview.config.php | 68 +++++----- tests/datasets/api.fixture.php | 50 +++---- update.php | 24 ++++ 15 files changed, 447 insertions(+), 431 deletions(-) delete mode 100644 src/Model/ItemContent.php create mode 100644 src/Model/Post/Content.php diff --git a/include/enotify.php b/include/enotify.php index 62e2e2669f..eb56603277 100644 --- a/include/enotify.php +++ b/include/enotify.php @@ -20,6 +20,7 @@ */ use Friendica\Content\Text\BBCode; +use Friendica\Content\Text\Plaintext; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\Renderer; @@ -28,7 +29,6 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Item; -use Friendica\Model\ItemContent; use Friendica\Model\Notification; use Friendica\Model\Post; use Friendica\Model\User; @@ -176,7 +176,7 @@ function notification($params) $item_post_type = Item::postType($item); - $content = ItemContent::getPlaintextPost($item, 70); + $content = Plaintext::getPost($item, 70); if (!empty($content['text'])) { $title = '"' . trim(str_replace("\n", " ", $content['text'])) . '"'; } else { diff --git a/src/Content/Text/Plaintext.php b/src/Content/Text/Plaintext.php index d5c5ea9ce7..b6d767f159 100644 --- a/src/Content/Text/Plaintext.php +++ b/src/Content/Text/Plaintext.php @@ -21,6 +21,9 @@ namespace Friendica\Content\Text; +use Friendica\Core\Protocol; +use Friendica\DI; + class Plaintext { /** @@ -88,4 +91,162 @@ class Plaintext return $res; } + + /** + * Convert a message into plaintext for connectors to other networks + * + * @param array $item The message array that is about to be posted + * @param int $limit The maximum number of characters when posting to that network + * @param bool $includedlinks Has an attached link to be included into the message? + * @param int $htmlmode This controls the behavior of the BBCode conversion + * @param string $target_network Name of the network where the post should go to. + * + * @return array Same array structure than \Friendica\Content\Text\BBCode::getAttachedData + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @see \Friendica\Content\Text\BBCode::getAttachedData + * + */ + public static function getPost($item, $limit = 0, $includedlinks = false, $htmlmode = BBCode::API, $target_network = '') + { + // Remove hashtags + $URLSearchString = '^\[\]'; + $body = preg_replace("/([#@])\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '$1$3', $item['body']); + + // Add an URL element if the text contains a raw link + $body = preg_replace('/([^\]\=\'"]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism', + '$1[url]$2[/url]', $body); + + // Remove the abstract + $body = BBCode::stripAbstract($body); + + // At first look at data that is attached via "type-..." stuff + // This will hopefully replaced with a dedicated bbcode later + //$post = self::getAttachedData($b['body']); + $post = BBCode::getAttachedData($body, $item); + + if (($item['title'] != '') && ($post['text'] != '')) { + $post['text'] = trim($item['title'] . "\n\n" . $post['text']); + } elseif ($item['title'] != '') { + $post['text'] = trim($item['title']); + } + + $abstract = ''; + + // Fetch the abstract from the given target network + if ($target_network != '') { + $default_abstract = BBCode::getAbstract($item['body']); + $abstract = BBCode::getAbstract($item['body'], $target_network); + + // If we post to a network with no limit we only fetch + // an abstract exactly for this network + if (($limit == 0) && ($abstract == $default_abstract)) { + $abstract = ''; + } + } else {// Try to guess the correct target network + switch ($htmlmode) { + case BBCode::TWITTER: + $abstract = BBCode::getAbstract($item['body'], Protocol::TWITTER); + break; + + case BBCode::OSTATUS: + $abstract = BBCode::getAbstract($item['body'], Protocol::STATUSNET); + break; + + default: // We don't know the exact target. + // We fetch an abstract since there is a posting limit. + if ($limit > 0) { + $abstract = BBCode::getAbstract($item['body']); + } + } + } + + if ($abstract != '') { + $post['text'] = $abstract; + + if ($post['type'] == 'text') { + $post['type'] = 'link'; + $post['url'] = $item['plink']; + } + } + + $html = BBCode::convert($post['text'] . ($post['after'] ?? ''), false, $htmlmode); + $msg = HTML::toPlaintext($html, 0, true); + $msg = trim(html_entity_decode($msg, ENT_QUOTES, 'UTF-8')); + + $link = ''; + if ($includedlinks) { + if ($post['type'] == 'link') { + $link = $post['url']; + } elseif ($post['type'] == 'text') { + $link = $post['url'] ?? ''; + } elseif ($post['type'] == 'video') { + $link = $post['url']; + } elseif ($post['type'] == 'photo') { + $link = $post['image']; + } + + if (($msg == '') && isset($post['title'])) { + $msg = trim($post['title']); + } + + if (($msg == '') && isset($post['description'])) { + $msg = trim($post['description']); + } + + // If the link is already contained in the post, then it neeedn't to be added again + // But: if the link is beyond the limit, then it has to be added. + if (($link != '') && strstr($msg, $link)) { + $pos = strpos($msg, $link); + + // Will the text be shortened in the link? + // Or is the link the last item in the post? + if (($limit > 0) && ($pos < $limit) && (($pos + 23 > $limit) || ($pos + strlen($link) == strlen($msg)))) { + $msg = trim(str_replace($link, '', $msg)); + } elseif (($limit == 0) || ($pos < $limit)) { + // The limit has to be increased since it will be shortened - but not now + // Only do it with Twitter + if (($limit > 0) && (strlen($link) > 23) && ($htmlmode == BBCode::TWITTER)) { + $limit = $limit - 23 + strlen($link); + } + + $link = ''; + + if ($post['type'] == 'text') { + unset($post['url']); + } + } + } + } + + if ($limit > 0) { + // Reduce multiple spaces + // When posted to a network with limited space, we try to gain space where possible + while (strpos($msg, ' ') !== false) { + $msg = str_replace(' ', ' ', $msg); + } + + // Twitter is using its own limiter, so we always assume that shortened links will have this length + if (iconv_strlen($link, 'UTF-8') > 0) { + $limit = $limit - 23; + } + + if (iconv_strlen($msg, 'UTF-8') > $limit) { + if (($post['type'] == 'text') && isset($post['url'])) { + $post['url'] = $item['plink']; + } elseif (!isset($post['url'])) { + $limit = $limit - 23; + $post['url'] = $item['plink']; + } elseif (strpos($item['body'], '[share') !== false) { + $post['url'] = $item['plink']; + } elseif (DI::pConfig()->get($item['uid'], 'system', 'no_intelligent_shortening')) { + $post['url'] = $item['plink']; + } + $msg = self::shorten($msg, $limit); + } + } + + $post['text'] = trim($msg); + + return $post; + } } diff --git a/src/Core/Update.php b/src/Core/Update.php index 204c0d0ac3..404b217bba 100644 --- a/src/Core/Update.php +++ b/src/Core/Update.php @@ -72,8 +72,8 @@ class Update } } - // The postupdate has to completed version 1281 for the new post views to take over - $postupdate = DI::config()->get("system", "post_update_version"); + // The postupdate has to completed version 1288 for the new post views to take over + $postupdate = DI::config()->get("system", "post_update_version", NEW_TABLE_STRUCTURE_VERSION); if ($postupdate < NEW_TABLE_STRUCTURE_VERSION) { $error = DI::l10n()->t('Updates from postupdate version %s are not supported. Please update at least to version 2021.01 and wait until the postupdate finished version 1383.', $postupdate); if (DI::mode()->getExecutor() == Mode::INDEX) { diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php index f461cedd51..8a3d6cc80a 100644 --- a/src/Database/DBStructure.php +++ b/src/Database/DBStructure.php @@ -72,8 +72,14 @@ class DBStructure */ public static function dropTables(bool $execute) { + $postupdate = DI::config()->get("system", "post_update_version", PostUpdate::VERSION); + if ($postupdate < PostUpdate::VERSION) { + echo DI::l10n()->t('The post update is at version %d, it has to be at %d to safely drop the tables.', $postupdate, PostUpdate::VERSION); + return; + } + $old_tables = ['fserver', 'gcign', 'gcontact', 'gcontact-relation', 'gfollower' ,'glink', 'item-delivery-data', - 'item_id', 'poll', 'poll_result', 'queue', 'retriever_rule', 'sign', 'spam', 'term']; + 'item-activity', 'item-content', 'item_id', 'poll', 'poll_result', 'queue', 'retriever_rule', 'sign', 'spam', 'term']; $tables = DBA::selectToArray(['INFORMATION_SCHEMA' => 'TABLES'], ['TABLE_NAME'], ['TABLE_SCHEMA' => DBA::databaseName(), 'TABLE_TYPE' => 'BASE TABLE']); diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 2140aa837a..51ce0d2b55 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -230,6 +230,11 @@ class PostUpdate return true; } + if (!DBStructure::existsTable('item-content')) { + DI::config()->set('system', 'post_update_version', 1342); + return true; + } + $id = DI::config()->get('system', 'post_update_version_1341_id', 0); Logger::info('Start', ['item' => $id]); @@ -283,7 +288,7 @@ class PostUpdate return true; } - if (!DBStructure::existsTable('term')) { + if (!DBStructure::existsTable('term') || !DBStructure::existsTable('item-content')) { DI::config()->set('system', 'post_update_version', 1342); return true; } @@ -510,6 +515,11 @@ class PostUpdate return true; } + if (!DBStructure::existsTable('item-activity')) { + DI::config()->set('system', 'post_update_version', 1347); + return true; + } + $id = DI::config()->get("system", "post_update_version_1347_id", 0); Logger::info('Start', ['item' => $id]); diff --git a/src/Model/Item.php b/src/Model/Item.php index 39fa8963a0..cf844b1032 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -99,12 +99,12 @@ class Item 'author-id', 'author-link', 'owner-link', 'contact-uid', 'signed_text', 'network']; - // Field list for "item-content" table that is mixed with the item table + // Field list for "post-content" table that is mixed with the item table const MIXED_CONTENT_FIELDLIST = ['title', 'content-warning', 'body', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'verb', 'object-type', 'object', 'target-type', 'target', 'plink']; - // Field list for "item-content" table that is not present in the "item" table + // Field list for "post-content" table that is not present in the "item" table const CONTENT_FIELDLIST = ['language', 'raw-body']; // All fields in the item table @@ -133,7 +133,7 @@ class Item const PRIVATE = 1; const UNLISTED = 2; - const TABLES = ['item', 'user-item', 'item-content', 'post-delivery-data', 'diaspora-interaction']; + const TABLES = ['item', 'user-item', 'post-content', 'post-delivery-data', 'diaspora-interaction']; private static function getItemFields() { @@ -224,7 +224,6 @@ class Item // Remove all media attachments from the body and store them in the post-media table $content_fields['raw-body'] = Post\Media::insertFromBody($item['uri-id'], $content_fields['raw-body']); $content_fields['raw-body'] = self::setHashtags($content_fields['raw-body']); - self::updateContent($content_fields, ['uri-id' => $item['uri-id']]); } if (!empty($fields['file'])) { @@ -1054,9 +1053,9 @@ class Item $notify_type = Delivery::POST; } - if (!in_array($item['verb'], self::ACTIVITIES) && !self::insertContent($item)) { + if (!in_array($item['verb'], self::ACTIVITIES) && !Post\Content::insert($item['uri-id'], $item)) { // This shouldn't happen - Logger::warning('No content stored, quitting', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'causer-id' => ($item['causer-id'] ?? 0), 'post-type' => $item['post-type'], 'network' => $item['network']]); + Logger::warning('No post-content stored, quitting', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'causer-id' => ($item['causer-id'] ?? 0), 'post-type' => $item['post-type'], 'network' => $item['network']]); return 0; } @@ -1300,67 +1299,6 @@ class Item } } - /** - * Insert a new item content entry - * - * @param array $item The item fields that are to be inserted - * @return bool "true" if content was inserted or already existed - * @throws \Exception - */ - private static function insertContent(array $item) - { - $fields = ['uri-plink-hash' => (string)$item['uri-id'], 'uri-id' => $item['uri-id']]; - - foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) { - if (isset($item[$field])) { - $fields[$field] = $item[$field]; - } - } - - $found = DBA::exists('item-content', ['uri-id' => $item['uri-id']]); - if ($found) { - Logger::info('Existing content found', ['uri-id' => $item['uri-id'], 'uri' => $item['uri']]); - return true; - } - - DBA::insert('item-content', $fields, Database::INSERT_IGNORE); - - $found = DBA::exists('item-content', ['uri-id' => $item['uri-id']]); - if ($found) { - Logger::notice('Content inserted', ['uri-id' => $item['uri-id'], 'uri' => $item['uri']]); - return true; - } - - // This shouldn't happen. - Logger::error("Content wasn't inserted", $item); - return false; - } - - /** - * Update existing item content entries - * - * @param array $item The item fields that are to be changed - * @param array $condition The condition for finding the item content entries - * @throws \Exception - */ - private static function updateContent($item, $condition) - { - // We have to select only the fields from the "item-content" table - $fields = []; - foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) { - if (isset($item[$field])) { - $fields[$field] = $item[$field]; - } - } - - if (empty($fields)) { - return; - } - - DBA::update('item-content', $fields, $condition, true); - Logger::info('Updated content', ['condition' => $condition]); - } - /** * Distributes public items to the receivers * diff --git a/src/Model/ItemContent.php b/src/Model/ItemContent.php deleted file mode 100644 index 304ab25d97..0000000000 --- a/src/Model/ItemContent.php +++ /dev/null @@ -1,236 +0,0 @@ -. - * - */ - -namespace Friendica\Model; - -use Friendica\Content\Text; -use Friendica\Content\Text\BBCode; -use Friendica\Core\Protocol; -use Friendica\Database\DBA; -use Friendica\DI; - -class ItemContent -{ - /** - * Search posts for given content - * - * @param string $search - * @param integer $uid - * @param integer $start - * @param integer $limit - * @param integer $last_uriid - * @return array - */ - public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100, int $last_uriid = 0) - { - $condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE)) - AND (NOT `private` OR (`private` AND `uid` = ?)) - AND `uri-id` IN (SELECT `uri-id` FROM `post-view` WHERE `network` IN (?, ?, ?, ?))", - $search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS]; - - if (!empty($last_uriid)) { - $condition = DBA::mergeConditions($condition, ["`uri-id` < ?", $last_uriid]); - } - - $params = [ - 'order' => ['uri-id' => true], - 'group_by' => ['uri-id'], - 'limit' => [$start, $limit] - ]; - - $tags = Post::select(['uri-id'], $condition, $params); - - $uriids = []; - while ($tag = DBA::fetch($tags)) { - $uriids[] = $tag['uri-id']; - } - DBA::close($tags); - - return $uriids; - } - - public static function countBySearch(string $search, int $uid = 0) - { - $condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE)) - AND (NOT `private` OR (`private` AND `uid` = ?)) - AND `uri-id` IN (SELECT `uri-id` FROM `post-view` WHERE `network` IN (?, ?, ?, ?))", - $search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS]; - return Post::count($condition); - } - - /** - * Convert a message into plaintext for connectors to other networks - * - * @param array $item The message array that is about to be posted - * @param int $limit The maximum number of characters when posting to that network - * @param bool $includedlinks Has an attached link to be included into the message? - * @param int $htmlmode This controls the behavior of the BBCode conversion - * @param string $target_network Name of the network where the post should go to. - * - * @return array Same array structure than \Friendica\Content\Text\BBCode::getAttachedData - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - * @see \Friendica\Content\Text\BBCode::getAttachedData - * - */ - public static function getPlaintextPost($item, $limit = 0, $includedlinks = false, $htmlmode = BBCode::API, $target_network = '') - { - // Remove hashtags - $URLSearchString = '^\[\]'; - $body = preg_replace("/([#@])\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '$1$3', $item['body']); - - // Add an URL element if the text contains a raw link - $body = preg_replace('/([^\]\=\'"]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism', - '$1[url]$2[/url]', $body); - - // Remove the abstract - $body = Text\BBCode::stripAbstract($body); - - // At first look at data that is attached via "type-..." stuff - // This will hopefully replaced with a dedicated bbcode later - //$post = self::getAttachedData($b['body']); - $post = Text\BBCode::getAttachedData($body, $item); - - if (($item['title'] != '') && ($post['text'] != '')) { - $post['text'] = trim($item['title'] . "\n\n" . $post['text']); - } elseif ($item['title'] != '') { - $post['text'] = trim($item['title']); - } - - $abstract = ''; - - // Fetch the abstract from the given target network - if ($target_network != '') { - $default_abstract = Text\BBCode::getAbstract($item['body']); - $abstract = Text\BBCode::getAbstract($item['body'], $target_network); - - // If we post to a network with no limit we only fetch - // an abstract exactly for this network - if (($limit == 0) && ($abstract == $default_abstract)) { - $abstract = ''; - } - } else {// Try to guess the correct target network - switch ($htmlmode) { - case BBCode::TWITTER: - $abstract = Text\BBCode::getAbstract($item['body'], Protocol::TWITTER); - break; - - case BBCode::OSTATUS: - $abstract = Text\BBCode::getAbstract($item['body'], Protocol::STATUSNET); - break; - - default: // We don't know the exact target. - // We fetch an abstract since there is a posting limit. - if ($limit > 0) { - $abstract = Text\BBCode::getAbstract($item['body']); - } - } - } - - if ($abstract != '') { - $post['text'] = $abstract; - - if ($post['type'] == 'text') { - $post['type'] = 'link'; - $post['url'] = $item['plink']; - } - } - - $html = Text\BBCode::convert($post['text'] . ($post['after'] ?? ''), false, $htmlmode); - $msg = Text\HTML::toPlaintext($html, 0, true); - $msg = trim(html_entity_decode($msg, ENT_QUOTES, 'UTF-8')); - - $link = ''; - if ($includedlinks) { - if ($post['type'] == 'link') { - $link = $post['url']; - } elseif ($post['type'] == 'text') { - $link = $post['url'] ?? ''; - } elseif ($post['type'] == 'video') { - $link = $post['url']; - } elseif ($post['type'] == 'photo') { - $link = $post['image']; - } - - if (($msg == '') && isset($post['title'])) { - $msg = trim($post['title']); - } - - if (($msg == '') && isset($post['description'])) { - $msg = trim($post['description']); - } - - // If the link is already contained in the post, then it neeedn't to be added again - // But: if the link is beyond the limit, then it has to be added. - if (($link != '') && strstr($msg, $link)) { - $pos = strpos($msg, $link); - - // Will the text be shortened in the link? - // Or is the link the last item in the post? - if (($limit > 0) && ($pos < $limit) && (($pos + 23 > $limit) || ($pos + strlen($link) == strlen($msg)))) { - $msg = trim(str_replace($link, '', $msg)); - } elseif (($limit == 0) || ($pos < $limit)) { - // The limit has to be increased since it will be shortened - but not now - // Only do it with Twitter - if (($limit > 0) && (strlen($link) > 23) && ($htmlmode == BBCode::TWITTER)) { - $limit = $limit - 23 + strlen($link); - } - - $link = ''; - - if ($post['type'] == 'text') { - unset($post['url']); - } - } - } - } - - if ($limit > 0) { - // Reduce multiple spaces - // When posted to a network with limited space, we try to gain space where possible - while (strpos($msg, ' ') !== false) { - $msg = str_replace(' ', ' ', $msg); - } - - // Twitter is using its own limiter, so we always assume that shortened links will have this length - if (iconv_strlen($link, 'UTF-8') > 0) { - $limit = $limit - 23; - } - - if (iconv_strlen($msg, 'UTF-8') > $limit) { - if (($post['type'] == 'text') && isset($post['url'])) { - $post['url'] = $item['plink']; - } elseif (!isset($post['url'])) { - $limit = $limit - 23; - $post['url'] = $item['plink']; - } elseif (strpos($item['body'], '[share') !== false) { - $post['url'] = $item['plink']; - } elseif (DI::pConfig()->get($item['uid'], 'system', 'no_intelligent_shortening')) { - $post['url'] = $item['plink']; - } - $msg = Text\Plaintext::shorten($msg, $limit); - } - } - - $post['text'] = trim($msg); - - return $post; - } -} diff --git a/src/Model/Post.php b/src/Model/Post.php index b418c30e44..59e2f419c5 100644 --- a/src/Model/Post.php +++ b/src/Model/Post.php @@ -446,13 +446,13 @@ class Post $affected = DBA::affectedRows(); } - $update_fields = DBStructure::getFieldsForTable('item-content', $fields); + $update_fields = DBStructure::getFieldsForTable('post-content', $fields); if (!empty($update_fields)) { $rows = DBA::selectToArray('post-view', ['uri-id'], $condition, ['group_by' => ['uri-id']]); $uriids = array_column($rows, 'uri-id'); - if (!DBA::update('item-content', $update_fields, ['uri-id' => $uriids])) { + if (!DBA::update('post-content', $update_fields, ['uri-id' => $uriids])) { DBA::rollback(); - Logger::notice('Updating item-content failed', ['fields' => $update_fields, 'condition' => $condition]); + Logger::notice('Updating post-content failed', ['fields' => $update_fields, 'condition' => $condition]); return false; } $affected = max($affected, DBA::affectedRows()); diff --git a/src/Model/Post/Content.php b/src/Model/Post/Content.php new file mode 100644 index 0000000000..5d9aaf62d6 --- /dev/null +++ b/src/Model/Post/Content.php @@ -0,0 +1,147 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use \BadMethodCallException; +use Friendica\Core\Protocol; +use Friendica\Database\Database; +use Friendica\Database\DBA; +use Friendica\Database\DBStructure; +use Friendica\Model\Post; + +class Content +{ + /** + * Insert a new post-content entry + * + * @param integer $uri_id + * @param array $fields + * @return bool success + * @throws \Exception + */ + public static function insert(int $uri_id, array $data = []) + { + if (empty($uri_id)) { + throw new BadMethodCallException('Empty URI_id'); + } + + if (DBA::exists('post-content', ['uri-id' => $uri_id])) { + return false; + } + + $fields = DBStructure::getFieldsForTable('post-content', $data); + + // Additionally assign the key fields + $fields['uri-id'] = $uri_id; + + return DBA::insert('post-content', $fields, Database::INSERT_IGNORE); + } + + /** + * Update a post content entry + * + * @param integer $uri_id + * @param array $data + * @param bool $insert_if_missing + * @return bool + * @throws \Exception + */ + public static function update(int $uri_id, array $data = [], bool $insert_if_missing = false) + { + if (empty($uri_id)) { + throw new BadMethodCallException('Empty URI_id'); + } + + $fields = DBStructure::getFieldsForTable('post-content', $data); + + // Remove the key fields + unset($fields['uri-id']); + + if (empty($fields)) { + return true; + } + + return DBA::update('post-content', $fields, ['uri-id' => $uri_id], $insert_if_missing ? true : []); + } + + /** + * Delete a row from the post-content table + * + * @param array $conditions Field condition(s) + * @param array $options + * - cascade: If true we delete records in other tables that depend on the one we're deleting through + * relations (default: true) + * + * @return boolean was the delete successful? + * @throws \Exception + */ + public static function delete(array $conditions, array $options = []) + { + return DBA::delete('post-content', $conditions, $options); + } + + + /** + * Search posts for given content + * + * @param string $search + * @param integer $uid + * @param integer $start + * @param integer $limit + * @param integer $last_uriid + * @return array + */ + public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100, int $last_uriid = 0) + { + $condition = ["`uri-id` IN (SELECT `uri-id` FROM `post-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE)) + AND (NOT `private` OR (`private` AND `uid` = ?)) AND (`network` IN (?, ?, ?, ?) OR (`uid` = ? AND `uid` != ?))", + $search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, $uid, 0]; + + if (!empty($last_uriid)) { + $condition = DBA::mergeConditions($condition, ["`uri-id` < ?", $last_uriid]); + } + + $params = [ + 'order' => ['uri-id' => true], + 'group_by' => ['uri-id'], + 'limit' => [$start, $limit] + ]; + + $tags = Post::select(['uri-id'], $condition, $params); + + $uriids = []; + while ($tag = DBA::fetch($tags)) { + $uriids[] = $tag['uri-id']; + } + DBA::close($tags); + + return $uriids; + } + + public static function countBySearch(string $search, int $uid = 0) + { + $condition = ["`uri-id` IN (SELECT `uri-id` FROM `post-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE)) + AND (NOT `private` OR (`private` AND `uid` = ?)) AND (`network` IN (?, ?, ?, ?) OR (`uid` = ? AND `uid` != ?))", + $search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, $uid, 0]; + return Post::count($condition); + } +} diff --git a/src/Module/Search/Index.php b/src/Module/Search/Index.php index 1c55ff4b4b..78f200810c 100644 --- a/src/Module/Search/Index.php +++ b/src/Module/Search/Index.php @@ -166,8 +166,8 @@ class Index extends BaseSearch $count = Tag::countByTag($search, local_user()); } else { Logger::info('Start fulltext search.', ['q' => $search]); - $uriids = ItemContent::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage(), $last_uriid); - $count = ItemContent::countBySearch($search, local_user()); + $uriids = Post\Content::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage(), $last_uriid); + $count = Post\Content::countBySearch($search, local_user()); } if (!empty($uriids)) { diff --git a/src/Worker/Expire.php b/src/Worker/Expire.php index 52e0912f52..2c419c401a 100644 --- a/src/Worker/Expire.php +++ b/src/Worker/Expire.php @@ -55,8 +55,8 @@ class Expire // Normally we shouldn't have orphaned data at all. // If we do have some, then we have to check why. Logger::log('Deleting orphaned item content - start', Logger::DEBUG); - $condition = ["NOT EXISTS (SELECT `uri-id` FROM `item` WHERE `item`.`uri-id` = `item-content`.`uri-id`)"]; - DBA::delete('item-content', $condition); + $condition = ["NOT EXISTS (SELECT `uri-id` FROM `item` WHERE `item`.`uri-id` = `post-content`.`uri-id`)"]; + DBA::delete('post-content', $condition); Logger::log('Orphaned item content deleted: ' . DBA::affectedRows(), Logger::DEBUG); // make this optional as it could have a performance impact on large sites diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 4a0f730f3f..d0c35e4f8d 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1395); + define('DB_UPDATE_VERSION', 1396); } return [ @@ -858,55 +858,6 @@ return [ "causer-id" => ["causer-id"], ] ], - "item-activity" => [ - "comment" => "Activities for items", - "fields" => [ - "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"], - "uri" => ["type" => "varchar(255)", "comment" => ""], - "uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], - "uri-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"], - "activity" => ["type" => "smallint unsigned", "not null" => "1", "default" => "0", "comment" => ""] - ], - "indexes" => [ - "PRIMARY" => ["id"], - "uri-hash" => ["UNIQUE", "uri-hash"], - "uri" => ["uri(191)"], - "uri-id" => ["uri-id"] - ] - ], - "item-content" => [ - "comment" => "Content for all posts", - "fields" => [ - "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"], - "uri" => ["type" => "varchar(255)", "comment" => ""], - "uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], - "uri-plink-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"], - "title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"], - "content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "body" => ["type" => "mediumtext", "comment" => "item body content"], - "raw-body" => ["type" => "mediumtext", "comment" => "Body without embedded media links"], - "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"], - "coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"], - "language" => ["type" => "text", "comment" => "Language information about this post"], - "app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"], - "rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""], - "rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"], - "object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"], - "object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"], - "target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"], - "target" => ["type" => "text", "comment" => "JSON encoded target structure if used"], - "plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"], - "verb" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams verb"] - ], - "indexes" => [ - "PRIMARY" => ["id"], - "uri-plink-hash" => ["UNIQUE", "uri-plink-hash"], - "title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"], - "uri" => ["uri(191)"], - "plink" => ["plink(191)"], - "uri-id" => ["uri-id"] - ] - ], "locks" => [ "comment" => "", "fields" => [ @@ -1161,6 +1112,33 @@ return [ "uid" => ["uid"], ] ], + "post-content" => [ + "comment" => "Content for all posts", + "fields" => [ + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"], + "content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "body" => ["type" => "mediumtext", "comment" => "item body content"], + "raw-body" => ["type" => "mediumtext", "comment" => "Body without embedded media links"], + "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"], + "coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"], + "language" => ["type" => "text", "comment" => "Language information about this post"], + "app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"], + "rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""], + "rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"], + "object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"], + "object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"], + "target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"], + "target" => ["type" => "text", "comment" => "JSON encoded target structure if used"], + "resource-id" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type"], + "plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"] + ], + "indexes" => [ + "PRIMARY" => ["uri-id"], + "plink" => ["plink(191)"], + "title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"], + ] + ], "post-delivery-data" => [ "comment" => "Delivery data for items", "fields" => [ diff --git a/static/dbview.config.php b/static/dbview.config.php index 5d1231d5a6..c7f52a5cf4 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -60,7 +60,6 @@ "commented" => ["item", "commented"], "received" => ["item", "received"], "changed" => ["item", "changed"], - "resource-id" => ["item", "resource-id"], "post-type" => ["item", "post-type"], "private" => ["item", "private"], "pubmail" => ["item", "pubmail"], @@ -78,21 +77,22 @@ "vid" => ["item", "vid"], "psid" => ["item", "psid"], "verb" => "IF (`item`.`vid` IS NULL, '', `verb`.`name`)", - "title" => ["item-content", "title"], - "content-warning" => ["item-content", "content-warning"], - "raw-body" => ["item-content", "raw-body"], - "body" => ["item-content", "body"], - "rendered-hash" => ["item-content", "rendered-hash"], - "rendered-html" => ["item-content", "rendered-html"], - "language" => ["item-content", "language"], - "plink" => ["item-content", "plink"], - "location" => ["item-content", "location"], - "coord" => ["item-content", "coord"], - "app" => ["item-content", "app"], - "object-type" => ["item-content", "object-type"], - "object" => ["item-content", "object"], - "target-type" => ["item-content", "target-type"], - "target" => ["item-content", "target"], + "title" => ["post-content", "title"], + "content-warning" => ["post-content", "content-warning"], + "raw-body" => ["post-content", "raw-body"], + "body" => ["post-content", "body"], + "rendered-hash" => ["post-content", "rendered-hash"], + "rendered-html" => ["post-content", "rendered-html"], + "language" => ["post-content", "language"], + "plink" => ["post-content", "plink"], + "location" => ["post-content", "location"], + "coord" => ["post-content", "coord"], + "app" => ["post-content", "app"], + "object-type" => ["post-content", "object-type"], + "object" => ["post-content", "object"], + "target-type" => ["post-content", "target-type"], + "target" => ["post-content", "target"], + "resource-id" => ["post-content", "resource-id"], "contact-id" => ["item", "contact-id"], "contact-link" => ["contact", "url"], "contact-addr" => ["contact", "addr"], @@ -184,7 +184,7 @@ LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid` LEFT JOIN `event` ON `event`.`id` = `item`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `item`.`uri-id` - LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id` + LEFT JOIN `post-content` ON `post-content`.`uri-id` = `item`.`uri-id` LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `item`.`uri-id` AND `item`.`origin` LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid` STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`uri-id` = `item`.`parent-uri-id` AND `parent-item`.`uid` = `item`.`uid` @@ -213,7 +213,6 @@ "commented" => ["thread", "commented"], "received" => ["thread", "received"], "changed" => ["thread", "changed"], - "resource-id" => ["item", "resource-id"], "post-type" => ["thread", "post-type"], "private" => ["thread", "private"], "pubmail" => ["thread", "pubmail"], @@ -232,21 +231,22 @@ "vid" => ["item", "vid"], "psid" => ["item", "psid"], "verb" => "IF (`item`.`vid` IS NULL, '', `verb`.`name`)", - "title" => ["item-content", "title"], - "content-warning" => ["item-content", "content-warning"], - "raw-body" => ["item-content", "raw-body"], - "body" => ["item-content", "body"], - "rendered-hash" => ["item-content", "rendered-hash"], - "rendered-html" => ["item-content", "rendered-html"], - "language" => ["item-content", "language"], - "plink" => ["item-content", "plink"], - "location" => ["item-content", "location"], - "coord" => ["item-content", "coord"], - "app" => ["item-content", "app"], - "object-type" => ["item-content", "object-type"], - "object" => ["item-content", "object"], - "target-type" => ["item-content", "target-type"], - "target" => ["item-content", "target"], + "title" => ["post-content", "title"], + "content-warning" => ["post-content", "content-warning"], + "raw-body" => ["post-content", "raw-body"], + "body" => ["post-content", "body"], + "rendered-hash" => ["post-content", "rendered-hash"], + "rendered-html" => ["post-content", "rendered-html"], + "language" => ["post-content", "language"], + "plink" => ["post-content", "plink"], + "location" => ["post-content", "location"], + "coord" => ["post-content", "coord"], + "app" => ["post-content", "app"], + "object-type" => ["post-content", "object-type"], + "object" => ["post-content", "object"], + "target-type" => ["post-content", "target-type"], + "target" => ["post-content", "target"], + "resource-id" => ["post-content", "resource-id"], "contact-id" => ["thread", "contact-id"], "contact-link" => ["contact", "url"], "contact-addr" => ["contact", "addr"], @@ -338,7 +338,7 @@ LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid` LEFT JOIN `event` ON `event`.`id` = `item`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `thread`.`uri-id` - LEFT JOIN `item-content` ON `item-content`.`uri-id` = `thread`.`uri-id` + LEFT JOIN `post-content` ON `post-content`.`uri-id` = `thread`.`uri-id` LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `thread`.`uri-id` AND `thread`.`origin` LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid` STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`id` = `item`.`parent` diff --git a/tests/datasets/api.fixture.php b/tests/datasets/api.fixture.php index 89a3f8dba1..aa99fc7cb3 100644 --- a/tests/datasets/api.fixture.php +++ b/tests/datasets/api.fixture.php @@ -179,48 +179,36 @@ return [ 'guid' => '6', ], ], - 'item-content' => [ + 'post-content' => [ [ - 'id' => 1, - 'uri-id' => 1, - 'uri-plink-hash' => '1', - 'body' => 'Parent status', - 'plink' => 'http://localhost/display/1', + 'uri-id' => 1, + 'body' => 'Parent status', + 'plink' => 'http://localhost/display/1', ], [ - 'id' => 2, - 'uri-id' => 2, - 'uri-plink-hash' => '2', - 'body' => 'Reply', - 'plink' => 'http://localhost/display/2', + 'uri-id' => 2, + 'body' => 'Reply', + 'plink' => 'http://localhost/display/2', ], [ - 'id' => 3, - 'uri-id' => 3, - 'uri-plink-hash' => '3', - 'body' => 'Other user status', - 'plink' => 'http://localhost/display/3', + 'uri-id' => 3, + 'body' => 'Other user status', + 'plink' => 'http://localhost/display/3', ], [ - 'id' => 4, - 'uri-id' => 4, - 'uri-plink-hash' => '4', - 'body' => 'Friend user reply', - 'plink' => 'http://localhost/display/4', + 'uri-id' => 4, + 'body' => 'Friend user reply', + 'plink' => 'http://localhost/display/4', ], [ - 'id' => 5, - 'uri-id' => 5, - 'uri-plink-hash' => '5', - 'body' => '[share]Shared status[/share]', - 'plink' => 'http://localhost/display/5', + 'uri-id' => 5, + 'body' => '[share]Shared status[/share]', + 'plink' => 'http://localhost/display/5', ], [ - 'id' => 6, - 'uri-id' => 6, - 'uri-plink-hash' => '6', - 'body' => 'Friend user status', - 'plink' => 'http://localhost/display/6', + 'uri-id' => 6, + 'body' => 'Friend user status', + 'plink' => 'http://localhost/display/6', ], ], 'post-user' => [ diff --git a/update.php b/update.php index e8f8da9fb5..3e9292f2e4 100644 --- a/update.php +++ b/update.php @@ -266,6 +266,10 @@ function update_1348() function update_1349() { + if (!DBStructure::existsTable('item-activity')) { + return Update::SUCCESS; + } + $correct = true; foreach (Item::ACTIVITIES as $index => $activity) { if (!DBA::exists('verb', ['id' => $index + 1, 'name' => $activity])) { @@ -690,3 +694,23 @@ function update_1395() } return Update::SUCCESS; } + +function update_1396() +{ + if (!DBStructure::existsTable('item-content')) { + return Update::SUCCESS; + } + + if (!DBA::e("INSERT IGNORE INTO `post-content`(`uri-id`, `title`, `content-warning`, `body`, `raw-body`, + `location`, `coord`, `language`, `app`, `rendered-hash`, `rendered-html`, + `object-type`, `object`, `target-type`, `target`, `resource-id`, `plink`) + SELECT `item-content`.`uri-id`, `item-content`.`title`, `item-content`.`content-warning`, + `item-content`.`body`, `item-content`.`raw-body`, `item-content`.`location`, `item-content`.`coord`, + `item-content`.`language`, `item-content`.`app`, `item-content`.`rendered-hash`, + `item-content`.`rendered-html`, `item-content`.`object-type`, `item-content`.`object`, + `item-content`.`target-type`, `item-content`.`target`, `item`.`resource-id`, `item-content`.`plink` + FROM `item-content` INNER JOIN `item` ON `item`.`uri-id` = `item-content`.`uri-id`")) { + return Update::FAILED; + } + return Update::SUCCESS; +} \ No newline at end of file