Fix fulltext earch

- Add fulltext index on multiple columns in the profile table
- Remove "IN BOOLEAN MODE"
- Add documentation about ft_min_word_len
This commit is contained in:
Hypolite Petovan 2017-04-20 22:24:17 -04:00
parent 280d55f183
commit 3095c9cad3
3 changed files with 267 additions and 127 deletions

View File

@ -109,3 +109,15 @@ You can check the backlog of this queue at the `/admin` page.
11. Should there somehow have been an error at this point such as that there is no profile ID known.
Everything will get deleted based on the original `?url=` parameter.
## Note about search
The Directory uses MySQL fulltext capabilities to index profiles and offer a search feature.
However, the default minimum word size MySQL will index is 4, which ignores words like `PHP` and `USA`.
To index words smaller than 4 characters, you will have to edit your my.cnf/my.ini file to include this:
````
[mysqld]
ft_min_word_len = 3
````

View File

@ -1,18 +1,7 @@
-- phpMyAdmin SQL Dump
-- version 3.3.10.4
-- http://www.phpmyadmin.net
--
-- Generation Time: May 15, 2012 at 11:03 PM
-- Server version: 5.1.53
-- PHP Version: 5.3.5
-- Generation Time: Apr 21, 2017 at 03:58 AM
SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO";
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
SET time_zone = "+00:00";
--
--
@ -24,12 +13,11 @@ SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO";
--
CREATE TABLE IF NOT EXISTS `flag` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`id` int(11) NOT NULL,
`pid` int(11) NOT NULL,
`reason` int(11) NOT NULL,
`total` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
`total` int(11) NOT NULL DEFAULT '0'
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -38,12 +26,11 @@ CREATE TABLE IF NOT EXISTS `flag` (
--
CREATE TABLE IF NOT EXISTS `photo` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id` int(10) unsigned NOT NULL,
`profile-id` int(11) NOT NULL,
`data` mediumblob NOT NULL,
`score` float NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`score` float NOT NULL DEFAULT '0'
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -52,7 +39,7 @@ CREATE TABLE IF NOT EXISTS `photo` (
--
CREATE TABLE IF NOT EXISTS `profile` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`id` int(11) NOT NULL,
`name` char(255) NOT NULL,
`nurl` char(255) NOT NULL,
`comm` tinyint(1) NOT NULL DEFAULT '0',
@ -66,18 +53,8 @@ CREATE TABLE IF NOT EXISTS `profile` (
`tags` mediumtext NOT NULL,
`created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`censored` tinyint(4) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
KEY `name` (`name`),
KEY `nurl` (`nurl`),
KEY `comm` (`comm`),
KEY `pdesc` (`pdesc`),
KEY `locality` (`locality`),
KEY `region` (`region`),
KEY `country-name` (`country-name`),
KEY `homepage` (`homepage`),
FULLTEXT KEY `tags` (`tags`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`censored` tinyint(4) NOT NULL DEFAULT '0'
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -86,14 +63,11 @@ CREATE TABLE IF NOT EXISTS `profile` (
--
CREATE TABLE IF NOT EXISTS `session` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`id` bigint(20) unsigned NOT NULL,
`sid` char(255) NOT NULL,
`data` text NOT NULL,
`expire` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`),
KEY `sid` (`sid`),
KEY `expire` (`expire`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`expire` int(10) unsigned NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -102,7 +76,7 @@ CREATE TABLE IF NOT EXISTS `session` (
--
CREATE TABLE IF NOT EXISTS `site` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`id` int(11) NOT NULL,
`name` char(255) NOT NULL,
`url` char(255) NOT NULL,
`version` char(16) NOT NULL,
@ -111,37 +85,8 @@ CREATE TABLE IF NOT EXISTS `site` (
`info` text NOT NULL,
`admin_name` char(255) NOT NULL,
`admin_profile` char(255) NOT NULL,
`updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
-- --------------------------------------------------------
--
-- Table structure for table `tag`
--
CREATE TABLE IF NOT EXISTS `tag` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`term` char(255) NOT NULL,
`nurl` char(255) NOT NULL,
PRIMARY KEY (`id`),
KEY `term` (`term`),
KEY `nurl` (`nurl`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
-- --------------------------------------------------------
--
-- Table structure for table `user`
--
CREATE TABLE IF NOT EXISTS `user` (
`uid` int(11) NOT NULL AUTO_INCREMENT,
`email` char(255) NOT NULL,
`password` char(255) NOT NULL,
PRIMARY KEY (`uid`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00'
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -150,78 +95,262 @@ CREATE TABLE IF NOT EXISTS `user` (
--
CREATE TABLE IF NOT EXISTS `site-health` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id` int(10) unsigned NOT NULL,
`base_url` varchar(255) NOT NULL,
`effective_base_url` varchar(255) NULL DEFAULT NULL,
`health_score` int(11) NOT NULL DEFAULT 0,
`no_scrape_url` varchar(255) NULL DEFAULT NULL,
`effective_base_url` varchar(255) DEFAULT NULL,
`health_score` int(11) NOT NULL DEFAULT '0',
`no_scrape_url` varchar(255) DEFAULT NULL,
`dt_first_noticed` datetime NOT NULL,
`dt_last_seen` datetime NULL DEFAULT NULL,
`dt_last_probed` datetime NULL DEFAULT NULL,
`dt_last_heartbeat` datetime NULL DEFAULT NULL,
`name` varchar(255) NULL DEFAULT NULL,
`version` varchar(255) NULL DEFAULT NULL,
`plugins` text NULL DEFAULT NULL,
`reg_policy` char(32) NULL DEFAULT NULL,
`info` text NULL DEFAULT NULL,
`admin_name` varchar(255) NULL DEFAULT NULL,
`admin_profile` varchar(255) NULL DEFAULT NULL,
`ssl_state` bit(1) NULL DEFAULT NULL,
`ssl_grade` varchar(3) NULL DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `base_url` (`base_url`),
KEY `health_score` (`health_score`),
KEY `dt_last_seen` (`dt_last_seen`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`dt_last_seen` datetime DEFAULT NULL,
`dt_last_probed` datetime DEFAULT NULL,
`dt_last_heartbeat` datetime DEFAULT NULL,
`name` varchar(255) DEFAULT NULL,
`version` varchar(255) DEFAULT NULL,
`plugins` text,
`reg_policy` char(32) DEFAULT NULL,
`info` text,
`admin_name` varchar(255) DEFAULT NULL,
`admin_profile` varchar(255) DEFAULT NULL,
`ssl_state` bit(1) DEFAULT NULL,
`ssl_grade` varchar(3) DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `site-probe`
--
CREATE TABLE IF NOT EXISTS `site-probe` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id` int(10) unsigned NOT NULL,
`site_health_id` int(10) unsigned NOT NULL,
`dt_performed` datetime NOT NULL,
`request_time` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`),
KEY `site_health_id` (`site_health_id`),
KEY `dt_performed` (`dt_performed`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`request_time` int(10) unsigned NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `site-scrape`
--
CREATE TABLE IF NOT EXISTS `site-scrape` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id` int(10) unsigned NOT NULL,
`site_health_id` int(10) unsigned NOT NULL,
`dt_performed` datetime NOT NULL,
`request_time` int(10) unsigned NOT NULL,
`scrape_time` int(10) unsigned NOT NULL,
`photo_time` int(10) unsigned NOT NULL,
`total_time` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`),
KEY `site_health_id` (`site_health_id`),
KEY `dt_performed` (`dt_performed`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`total_time` int(10) unsigned NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `sync-pull-queue`
--
CREATE TABLE IF NOT EXISTS `sync-pull-queue` (
`url` varchar(255) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `sync-push-queue`
--
CREATE TABLE IF NOT EXISTS `sync-push-queue` (
`url` varchar(255) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `sync-targets`
--
CREATE TABLE IF NOT EXISTS `sync-targets` (
`base_url` varchar(255) NOT NULL,
`pull` bit(1) NOT NULL DEFAULT b'0',
`push` bit(1) NOT NULL DEFAULT b'1',
`dt_last_pull` bigint unsigned NULL DEFAULT NULL,
PRIMARY KEY (`base_url`),
KEY `push` (`push`),
KEY `pull` (`pull`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`dt_last_pull` bigint(20) unsigned DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
CREATE TABLE IF NOT EXISTS `sync-push-queue` (
`url` varchar(255) NOT NULL,
PRIMARY KEY (`url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ;
-- --------------------------------------------------------
CREATE TABLE IF NOT EXISTS `sync-pull-queue` (
`url` varchar(255) NOT NULL,
PRIMARY KEY (`url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ;
--
-- Table structure for table `sync-timestamps`
--
CREATE TABLE IF NOT EXISTS `sync-timestamps` (
`url` varchar(255) NOT NULL,
`modified` datetime NOT NULL,
PRIMARY KEY (`url`),
KEY `modified` (`modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ;
`modified` datetime NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `tag`
--
CREATE TABLE IF NOT EXISTS `tag` (
`id` int(11) NOT NULL,
`term` char(255) NOT NULL,
`nurl` char(255) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `user`
--
CREATE TABLE IF NOT EXISTS `user` (
`uid` int(11) NOT NULL,
`email` char(255) NOT NULL,
`password` char(255) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
--
-- Indexes for dumped tables
--
--
-- Indexes for table `flag`
--
ALTER TABLE `flag`
ADD PRIMARY KEY (`id`);
--
-- Indexes for table `photo`
--
ALTER TABLE `photo`
ADD PRIMARY KEY (`id`);
--
-- Indexes for table `profile`
--
ALTER TABLE `profile`
ADD PRIMARY KEY (`id`), ADD KEY `name` (`name`), ADD KEY `nurl` (`nurl`), ADD KEY `comm` (`comm`), ADD KEY `pdesc` (`pdesc`), ADD KEY `locality` (`locality`), ADD KEY `region` (`region`), ADD KEY `country-name` (`country-name`), ADD KEY `homepage` (`homepage`), ADD FULLTEXT KEY `tags` (`tags`), ADD FULLTEXT KEY `profile-ft` (`name`,`pdesc`,`homepage`,`locality`,`region`,`country-name`,`tags`);
--
-- Indexes for table `session`
--
ALTER TABLE `session`
ADD PRIMARY KEY (`id`), ADD KEY `sid` (`sid`), ADD KEY `expire` (`expire`);
--
-- Indexes for table `site`
--
ALTER TABLE `site`
ADD PRIMARY KEY (`id`);
--
-- Indexes for table `site-health`
--
ALTER TABLE `site-health`
ADD PRIMARY KEY (`id`), ADD KEY `base_url` (`base_url`), ADD KEY `health_score` (`health_score`), ADD KEY `dt_last_seen` (`dt_last_seen`);
--
-- Indexes for table `site-probe`
--
ALTER TABLE `site-probe`
ADD PRIMARY KEY (`id`), ADD KEY `site_health_id` (`site_health_id`), ADD KEY `dt_performed` (`dt_performed`);
--
-- Indexes for table `site-scrape`
--
ALTER TABLE `site-scrape`
ADD PRIMARY KEY (`id`), ADD KEY `site_health_id` (`site_health_id`), ADD KEY `dt_performed` (`dt_performed`);
--
-- Indexes for table `sync-pull-queue`
--
ALTER TABLE `sync-pull-queue`
ADD PRIMARY KEY (`url`);
--
-- Indexes for table `sync-push-queue`
--
ALTER TABLE `sync-push-queue`
ADD PRIMARY KEY (`url`);
--
-- Indexes for table `sync-targets`
--
ALTER TABLE `sync-targets`
ADD PRIMARY KEY (`base_url`), ADD KEY `push` (`push`), ADD KEY `pull` (`pull`);
--
-- Indexes for table `sync-timestamps`
--
ALTER TABLE `sync-timestamps`
ADD PRIMARY KEY (`url`), ADD KEY `modified` (`modified`);
--
-- Indexes for table `tag`
--
ALTER TABLE `tag`
ADD PRIMARY KEY (`id`), ADD KEY `term` (`term`), ADD KEY `nurl` (`nurl`);
--
-- Indexes for table `user`
--
ALTER TABLE `user`
ADD PRIMARY KEY (`uid`);
--
-- AUTO_INCREMENT for dumped tables
--
--
-- AUTO_INCREMENT for table `flag`
--
ALTER TABLE `flag`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6;
--
-- AUTO_INCREMENT for table `photo`
--
ALTER TABLE `photo`
MODIFY `id` int(10) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=1486;
--
-- AUTO_INCREMENT for table `profile`
--
ALTER TABLE `profile`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=1488;
--
-- AUTO_INCREMENT for table `session`
--
ALTER TABLE `session`
MODIFY `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=104;
--
-- AUTO_INCREMENT for table `site`
--
ALTER TABLE `site`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
--
-- AUTO_INCREMENT for table `site-health`
--
ALTER TABLE `site-health`
MODIFY `id` int(10) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=1513;
--
-- AUTO_INCREMENT for table `site-probe`
--
ALTER TABLE `site-probe`
MODIFY `id` int(10) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6651;
--
-- AUTO_INCREMENT for table `site-scrape`
--
ALTER TABLE `site-scrape`
MODIFY `id` int(10) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=21048;
--
-- AUTO_INCREMENT for table `tag`
--
ALTER TABLE `tag`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=2322;
--
-- AUTO_INCREMENT for table `user`
--
ALTER TABLE `user`
MODIFY `uid` int(11) NOT NULL AUTO_INCREMENT;

View File

@ -48,10 +48,9 @@ function directory_content(App $a)
));
if ($search) {
$search = dbesc($search . '*');
$search = dbesc($search);
}
$sql_extra = ((strlen($search)) ? " AND MATCH (`name`, `pdesc`, `homepage`, `locality`, `region`, `country-name`, `tags`)
AGAINST ('$search' IN BOOLEAN MODE) " : "");
$sql_extra = ((strlen($search)) ? " AND MATCH (`name`, `pdesc`, `homepage`, `locality`, `region`, `country-name`, `tags`) AGAINST ('$search') " : "");
if ($forums) {
$sql_extra .= " AND `comm` = 1 ";