Merge pull request #9234 from annando/issue-9231

Issue 9231: Speed up full text search

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
--- Friendica 2020.09-dev (Red Hot Poker)
+-- Friendica 2020.09-rc (Red Hot Poker)
--- DB_UPDATE_VERSION 1367
+-- DB_UPDATE_VERSION 1368
 -- ------------------------------------------
 
 
@@ -786,6 +786,7 @@ CREATE TABLE IF NOT EXISTS `item-content` (
 	`verb` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams verb',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `uri-plink-hash` (`uri-plink-hash`),
+	 FULLTEXT INDEX `title-content-warning-body` (`title`,`content-warning`,`body`),
 	 INDEX `uri` (`uri`(191)),
 	 INDEX `plink` (`plink`(191)),
 	 INDEX `uri-id` (`uri-id`),

src/Model/ItemContent.php

@@ -24,10 +24,39 @@ namespace Friendica\Model;
 use Friendica\Content\Text;
 use Friendica\Content\Text\BBCode;
 use Friendica\Core\Protocol;
+use Friendica\Database\DBA;
 use Friendica\DI;
 
 class ItemContent
 {
+	public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100)
+	{
+		$condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
+			AND (NOT `private` OR (`private` AND `uid` = ?))", $search, $uid];
+		$params = [
+			'order' => ['uri-id' => true],
+			'group_by' => ['uri-id'],
+			'limit' => [$start, $limit]
+		];
+
+		$tags = DBA::select('item', ['uri-id'], $condition, $params);
+
+		$uriids = [];
+		while ($tag = DBA::fetch($tags)) {
+			$uriids[] = $tag['uri-id'];
+		}
+		DBA::close($tags);
+
+		return $uriids;
+	}
+
+	public static function countBySearch(string $search, int $uid = 0)
+	{
+		$condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
+			AND (NOT `private` OR (`private` AND `uid` = ?))", $search, $uid];
+		return DBA::count('item', $condition);
+	}
+
 	/**
 	 * Convert a message into plaintext for connectors to other networks
 	 *

src/Module/Search/Index.php

@@ -34,6 +34,7 @@ use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Item;
+use Friendica\Model\ItemContent;
 use Friendica\Model\Tag;
 use Friendica\Module\BaseSearch;
 use Friendica\Network\HTTPException;
@@ -151,30 +152,17 @@ class Index extends BaseSearch
 		if ($tag) {
 			Logger::info('Start tag search.', ['q' => $search]);
 			$uriids = Tag::getURIIdListByTag($search, local_user(), $pager->getStart(), $pager->getItemsPerPage());
-
+			$count = Tag::countByTag($search, local_user());
-			if (!empty($uriids)) {
-				$params = ['order' => ['id' => true], 'group_by' => ['uri-id']];
-				$items = Item::selectForUser(local_user(), [], ['uri-id' => $uriids], $params);
-				$r = Item::inArray($items);
-				$count = Tag::countByTag($search, local_user());
-			} else {
-				$count = 0;
-			}
 		} else {
 			Logger::info('Start fulltext search.', ['q' => $search]);
+			$uriids = ItemContent::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage());
+			$count = ItemContent::countBySearch($search, local_user());
+		}
 
-			$condition = [
+		if (!empty($uriids)) {
-				"(`uid` = 0 OR (`uid` = ? AND NOT `global`))
+			$params = ['order' => ['id' => true], 'group_by' => ['uri-id']];
-				AND `body` LIKE CONCAT('%',?,'%')",
+			$items = Item::selectForUser(local_user(), [], ['uri-id' => $uriids], $params);
-				local_user(), $search
-			];
-			$params = [
-				'order' => ['id' => true],
-				'limit' => [$pager->getStart(), $pager->getItemsPerPage()]
-			];
-			$items = Item::selectForUser(local_user(), [], $condition, $params);
 			$r = Item::inArray($items);
-			$count = DBA::count('item', $condition);
 		}
 
 		if (!DBA::isResult($r)) {

static/dbstructure.config.php

@@ -54,7 +54,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1367);
+	define('DB_UPDATE_VERSION', 1368);
 }
 
 return [
@@ -857,6 +857,7 @@ return [
 		"indexes" => [
 			"PRIMARY" => ["id"],
 			"uri-plink-hash" => ["UNIQUE", "uri-plink-hash"],
+			"title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"],
 			"uri" => ["uri(191)"],
 			"plink" => ["plink(191)"],
 			"uri-id" => ["uri-id"]