From dfeae25e6dd4edfd096afcb3e51d48315cba84d0 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 19 Sep 2020 08:26:50 +0000 Subject: [PATCH 1/4] Issue 9231: Speed up full text search --- database.sql | 5 +++-- src/Model/ItemContent.php | 22 ++++++++++++++++++++++ src/Module/Search/Index.php | 29 +++++++++-------------------- static/dbstructure.config.php | 3 ++- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/database.sql b/database.sql index addd396ffd..f98b74eda4 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ --- Friendica 2020.09-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1367 +-- Friendica 2020.09-rc (Red Hot Poker) +-- DB_UPDATE_VERSION 1368 -- ------------------------------------------ @@ -786,6 +786,7 @@ CREATE TABLE IF NOT EXISTS `item-content` ( `verb` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams verb', PRIMARY KEY(`id`), UNIQUE INDEX `uri-plink-hash` (`uri-plink-hash`), + FULLTEXT INDEX `body` (`body`), INDEX `uri` (`uri`(191)), INDEX `plink` (`plink`(191)), INDEX `uri-id` (`uri-id`), diff --git a/src/Model/ItemContent.php b/src/Model/ItemContent.php index c8ad48ca46..c4e77cf413 100644 --- a/src/Model/ItemContent.php +++ b/src/Model/ItemContent.php @@ -24,10 +24,32 @@ namespace Friendica\Model; use Friendica\Content\Text; use Friendica\Content\Text\BBCode; use Friendica\Core\Protocol; +use Friendica\Database\DBA; use Friendica\DI; class ItemContent { + public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100) + { + $condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`body`) AGAINST (? IN BOOLEAN MODE)) + AND (NOT `private` OR (`private` AND `uid` = ?))", $search, $uid]; + $params = [ + 'order' => ['uri-id' => true], + 'group_by' => ['uri-id'], + 'limit' => [$start, $limit] + ]; + + $tags = DBA::select('item', ['uri-id'], $condition, $params); + + $uriids = []; + while ($tag = DBA::fetch($tags)) { + $uriids[] = $tag['uri-id']; + } + DBA::close($tags); + + return $uriids; + } + /** * Convert a message into plaintext for connectors to other networks * diff --git a/src/Module/Search/Index.php b/src/Module/Search/Index.php index 6cee581c2b..030b684e6b 100644 --- a/src/Module/Search/Index.php +++ b/src/Module/Search/Index.php @@ -34,6 +34,7 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Item; +use Friendica\Model\ItemContent; use Friendica\Model\Tag; use Friendica\Module\BaseSearch; use Friendica\Network\HTTPException; @@ -151,30 +152,18 @@ class Index extends BaseSearch if ($tag) { Logger::info('Start tag search.', ['q' => $search]); $uriids = Tag::getURIIdListByTag($search, local_user(), $pager->getStart(), $pager->getItemsPerPage()); - - if (!empty($uriids)) { - $params = ['order' => ['id' => true], 'group_by' => ['uri-id']]; - $items = Item::selectForUser(local_user(), [], ['uri-id' => $uriids], $params); - $r = Item::inArray($items); - $count = Tag::countByTag($search, local_user()); - } else { - $count = 0; - } } else { Logger::info('Start fulltext search.', ['q' => $search]); + $uriids = ItemContent::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage()); + } - $condition = [ - "(`uid` = 0 OR (`uid` = ? AND NOT `global`)) - AND `body` LIKE CONCAT('%',?,'%')", - local_user(), $search - ]; - $params = [ - 'order' => ['id' => true], - 'limit' => [$pager->getStart(), $pager->getItemsPerPage()] - ]; - $items = Item::selectForUser(local_user(), [], $condition, $params); + if (!empty($uriids)) { + $params = ['order' => ['id' => true], 'group_by' => ['uri-id']]; + $items = Item::selectForUser(local_user(), [], ['uri-id' => $uriids], $params); $r = Item::inArray($items); - $count = DBA::count('item', $condition); + $count = Tag::countByTag($search, local_user()); + } else { + $count = 0; } if (!DBA::isResult($r)) { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 23a299e652..c69da5a0ef 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -54,7 +54,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1367); + define('DB_UPDATE_VERSION', 1368); } return [ @@ -857,6 +857,7 @@ return [ "indexes" => [ "PRIMARY" => ["id"], "uri-plink-hash" => ["UNIQUE", "uri-plink-hash"], + "body" => ["FULLTEXT", "body"], "uri" => ["uri(191)"], "plink" => ["plink(191)"], "uri-id" => ["uri-id"] From 86592daaa16817b7554c243e59b0277006edf0fa Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 19 Sep 2020 10:45:11 +0000 Subject: [PATCH 2/4] Include more field in the fulltext search --- src/Model/ItemContent.php | 9 ++++++++- src/Module/Search/Index.php | 3 ++- static/dbstructure.config.php | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Model/ItemContent.php b/src/Model/ItemContent.php index c4e77cf413..daa2766e2f 100644 --- a/src/Model/ItemContent.php +++ b/src/Model/ItemContent.php @@ -31,7 +31,7 @@ class ItemContent { public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100) { - $condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`body`) AGAINST (? IN BOOLEAN MODE)) + $condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE)) AND (NOT `private` OR (`private` AND `uid` = ?))", $search, $uid]; $params = [ 'order' => ['uri-id' => true], @@ -50,6 +50,13 @@ class ItemContent return $uriids; } + public static function countBySearch(string $search, int $uid = 0) + { + $condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE)) + AND (NOT `private` OR (`private` AND `uid` = ?))", $search, $uid]; + return DBA::count('item', $condition); + } + /** * Convert a message into plaintext for connectors to other networks * diff --git a/src/Module/Search/Index.php b/src/Module/Search/Index.php index 030b684e6b..42d1d50c7b 100644 --- a/src/Module/Search/Index.php +++ b/src/Module/Search/Index.php @@ -152,16 +152,17 @@ class Index extends BaseSearch if ($tag) { Logger::info('Start tag search.', ['q' => $search]); $uriids = Tag::getURIIdListByTag($search, local_user(), $pager->getStart(), $pager->getItemsPerPage()); + $count = Tag::countByTag($search, local_user()); } else { Logger::info('Start fulltext search.', ['q' => $search]); $uriids = ItemContent::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage()); + $count = ItemContent::countBySearch($search, local_user()); } if (!empty($uriids)) { $params = ['order' => ['id' => true], 'group_by' => ['uri-id']]; $items = Item::selectForUser(local_user(), [], ['uri-id' => $uriids], $params); $r = Item::inArray($items); - $count = Tag::countByTag($search, local_user()); } else { $count = 0; } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index c69da5a0ef..ea506b4a92 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -857,7 +857,7 @@ return [ "indexes" => [ "PRIMARY" => ["id"], "uri-plink-hash" => ["UNIQUE", "uri-plink-hash"], - "body" => ["FULLTEXT", "body"], + "title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"], "uri" => ["uri(191)"], "plink" => ["plink(191)"], "uri-id" => ["uri-id"] From 6020e824a8ca76473a24e8ee363f3cbbad5c0339 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 19 Sep 2020 12:07:17 +0000 Subject: [PATCH 3/4] Don't set count to 0 --- src/Module/Search/Index.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Module/Search/Index.php b/src/Module/Search/Index.php index 42d1d50c7b..debaec7301 100644 --- a/src/Module/Search/Index.php +++ b/src/Module/Search/Index.php @@ -163,8 +163,6 @@ class Index extends BaseSearch $params = ['order' => ['id' => true], 'group_by' => ['uri-id']]; $items = Item::selectForUser(local_user(), [], ['uri-id' => $uriids], $params); $r = Item::inArray($items); - } else { - $count = 0; } if (!DBA::isResult($r)) { From 7fba4145ed5da78e5e68302cc25d1fab13a6fdee Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 19 Sep 2020 12:53:20 +0000 Subject: [PATCH 4/4] Updated database.sql --- database.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.sql b/database.sql index f98b74eda4..3b7ef341c1 100644 --- a/database.sql +++ b/database.sql @@ -786,7 +786,7 @@ CREATE TABLE IF NOT EXISTS `item-content` ( `verb` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams verb', PRIMARY KEY(`id`), UNIQUE INDEX `uri-plink-hash` (`uri-plink-hash`), - FULLTEXT INDEX `body` (`body`), + FULLTEXT INDEX `title-content-warning-body` (`title`,`content-warning`,`body`), INDEX `uri` (`uri`(191)), INDEX `plink` (`plink`(191)), INDEX `uri-id` (`uri-id`),