Fix fulltext earch

- Add fulltext index on multiple columns in the profile table
- Remove "IN BOOLEAN MODE"
- Add documentation about ft_min_word_len
This commit is contained in:
Hypolite Petovan 2017-04-20 22:24:17 -04:00
parent 280d55f183
commit 3095c9cad3
3 changed files with 267 additions and 127 deletions

View File

@ -108,4 +108,16 @@ You can check the backlog of this queue at the `/admin` page.
10. The `photo` provided will be downloaded and resized to 80x80, regardless of source size.
11. Should there somehow have been an error at this point such as that there is no profile ID known.
Everything will get deleted based on the original `?url=` parameter.
Everything will get deleted based on the original `?url=` parameter.
## Note about search
The Directory uses MySQL fulltext capabilities to index profiles and offer a search feature.
However, the default minimum word size MySQL will index is 4, which ignores words like `PHP` and `USA`.
To index words smaller than 4 characters, you will have to edit your my.cnf/my.ini file to include this:
````
[mysqld]
ft_min_word_len = 3
````

View File

@ -1,18 +1,7 @@
-- phpMyAdmin SQL Dump
-- version 3.3.10.4
-- http://www.phpmyadmin.net
--
-- Generation Time: May 15, 2012 at 11:03 PM
-- Server version: 5.1.53
-- PHP Version: 5.3.5
-- Generation Time: Apr 21, 2017 at 03:58 AM
SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO";
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
SET time_zone = "+00:00";
--
--
@ -24,12 +13,11 @@ SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO";
--
CREATE TABLE IF NOT EXISTS `flag` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`id` int(11) NOT NULL,
`pid` int(11) NOT NULL,
`reason` int(11) NOT NULL,
`total` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
`total` int(11) NOT NULL DEFAULT '0'
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -38,12 +26,11 @@ CREATE TABLE IF NOT EXISTS `flag` (
--
CREATE TABLE IF NOT EXISTS `photo` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id` int(10) unsigned NOT NULL,
`profile-id` int(11) NOT NULL,
`data` mediumblob NOT NULL,
`score` float NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`score` float NOT NULL DEFAULT '0'
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -52,7 +39,7 @@ CREATE TABLE IF NOT EXISTS `photo` (
--
CREATE TABLE IF NOT EXISTS `profile` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`id` int(11) NOT NULL,
`name` char(255) NOT NULL,
`nurl` char(255) NOT NULL,
`comm` tinyint(1) NOT NULL DEFAULT '0',
@ -66,18 +53,8 @@ CREATE TABLE IF NOT EXISTS `profile` (
`tags` mediumtext NOT NULL,
`created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`censored` tinyint(4) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
KEY `name` (`name`),
KEY `nurl` (`nurl`),
KEY `comm` (`comm`),
KEY `pdesc` (`pdesc`),
KEY `locality` (`locality`),
KEY `region` (`region`),
KEY `country-name` (`country-name`),
KEY `homepage` (`homepage`),
FULLTEXT KEY `tags` (`tags`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`censored` tinyint(4) NOT NULL DEFAULT '0'
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -86,14 +63,11 @@ CREATE TABLE IF NOT EXISTS `profile` (
--
CREATE TABLE IF NOT EXISTS `session` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`id` bigint(20) unsigned NOT NULL,
`sid` char(255) NOT NULL,
`data` text NOT NULL,
`expire` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`),
KEY `sid` (`sid`),
KEY `expire` (`expire`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`expire` int(10) unsigned NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -102,7 +76,7 @@ CREATE TABLE IF NOT EXISTS `session` (
--
CREATE TABLE IF NOT EXISTS `site` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`id` int(11) NOT NULL,
`name` char(255) NOT NULL,
`url` char(255) NOT NULL,
`version` char(16) NOT NULL,
@ -111,37 +85,8 @@ CREATE TABLE IF NOT EXISTS `site` (
`info` text NOT NULL,
`admin_name` char(255) NOT NULL,
`admin_profile` char(255) NOT NULL,
`updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
-- --------------------------------------------------------
--
-- Table structure for table `tag`
--
CREATE TABLE IF NOT EXISTS `tag` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`term` char(255) NOT NULL,
`nurl` char(255) NOT NULL,
PRIMARY KEY (`id`),
KEY `term` (`term`),
KEY `nurl` (`nurl`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
-- --------------------------------------------------------
--
-- Table structure for table `user`
--
CREATE TABLE IF NOT EXISTS `user` (
`uid` int(11) NOT NULL AUTO_INCREMENT,
`email` char(255) NOT NULL,
`password` char(255) NOT NULL,
PRIMARY KEY (`uid`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00'
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
@ -150,78 +95,262 @@ CREATE TABLE IF NOT EXISTS `user` (
--
CREATE TABLE IF NOT EXISTS `site-health` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id` int(10) unsigned NOT NULL,
`base_url` varchar(255) NOT NULL,
`effective_base_url` varchar(255) NULL DEFAULT NULL,
`health_score` int(11) NOT NULL DEFAULT 0,
`no_scrape_url` varchar(255) NULL DEFAULT NULL,
`effective_base_url` varchar(255) DEFAULT NULL,
`health_score` int(11) NOT NULL DEFAULT '0',
`no_scrape_url` varchar(255) DEFAULT NULL,
`dt_first_noticed` datetime NOT NULL,
`dt_last_seen` datetime NULL DEFAULT NULL,
`dt_last_probed` datetime NULL DEFAULT NULL,
`dt_last_heartbeat` datetime NULL DEFAULT NULL,
`name` varchar(255) NULL DEFAULT NULL,
`version` varchar(255) NULL DEFAULT NULL,
`plugins` text NULL DEFAULT NULL,
`reg_policy` char(32) NULL DEFAULT NULL,
`info` text NULL DEFAULT NULL,
`admin_name` varchar(255) NULL DEFAULT NULL,
`admin_profile` varchar(255) NULL DEFAULT NULL,
`ssl_state` bit(1) NULL DEFAULT NULL,
`ssl_grade` varchar(3) NULL DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `base_url` (`base_url`),
KEY `health_score` (`health_score`),
KEY `dt_last_seen` (`dt_last_seen`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`dt_last_seen` datetime DEFAULT NULL,
`dt_last_probed` datetime DEFAULT NULL,
`dt_last_heartbeat` datetime DEFAULT NULL,
`name` varchar(255) DEFAULT NULL,
`version` varchar(255) DEFAULT NULL,
`plugins` text,
`reg_policy` char(32) DEFAULT NULL,
`info` text,
`admin_name` varchar(255) DEFAULT NULL,
`admin_profile` varchar(255) DEFAULT NULL,
`ssl_state` bit(1) DEFAULT NULL,
`ssl_grade` varchar(3) DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `site-probe`
--
CREATE TABLE IF NOT EXISTS `site-probe` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id` int(10) unsigned NOT NULL,
`site_health_id` int(10) unsigned NOT NULL,
`dt_performed` datetime NOT NULL,
`request_time` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`),
KEY `site_health_id` (`site_health_id`),
KEY `dt_performed` (`dt_performed`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`request_time` int(10) unsigned NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `site-scrape`
--
CREATE TABLE IF NOT EXISTS `site-scrape` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id` int(10) unsigned NOT NULL,
`site_health_id` int(10) unsigned NOT NULL,
`dt_performed` datetime NOT NULL,
`request_time` int(10) unsigned NOT NULL,
`scrape_time` int(10) unsigned NOT NULL,
`photo_time` int(10) unsigned NOT NULL,
`total_time` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`),
KEY `site_health_id` (`site_health_id`),
KEY `dt_performed` (`dt_performed`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`total_time` int(10) unsigned NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `sync-pull-queue`
--
CREATE TABLE IF NOT EXISTS `sync-pull-queue` (
`url` varchar(255) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `sync-push-queue`
--
CREATE TABLE IF NOT EXISTS `sync-push-queue` (
`url` varchar(255) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `sync-targets`
--
CREATE TABLE IF NOT EXISTS `sync-targets` (
`base_url` varchar(255) NOT NULL,
`pull` bit(1) NOT NULL DEFAULT b'0',
`push` bit(1) NOT NULL DEFAULT b'1',
`dt_last_pull` bigint unsigned NULL DEFAULT NULL,
PRIMARY KEY (`base_url`),
KEY `push` (`push`),
KEY `pull` (`pull`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
`dt_last_pull` bigint(20) unsigned DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
CREATE TABLE IF NOT EXISTS `sync-push-queue` (
`url` varchar(255) NOT NULL,
PRIMARY KEY (`url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ;
-- --------------------------------------------------------
CREATE TABLE IF NOT EXISTS `sync-pull-queue` (
`url` varchar(255) NOT NULL,
PRIMARY KEY (`url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ;
--
-- Table structure for table `sync-timestamps`
--
CREATE TABLE IF NOT EXISTS `sync-timestamps` (
`url` varchar(255) NOT NULL,
`modified` datetime NOT NULL,
PRIMARY KEY (`url`),
KEY `modified` (`modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ;
`modified` datetime NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `tag`
--
CREATE TABLE IF NOT EXISTS `tag` (
`id` int(11) NOT NULL,
`term` char(255) NOT NULL,
`nurl` char(255) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `user`
--
CREATE TABLE IF NOT EXISTS `user` (
`uid` int(11) NOT NULL,
`email` char(255) NOT NULL,
`password` char(255) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
--
-- Indexes for dumped tables
--
--
-- Indexes for table `flag`
--
ALTER TABLE `flag`
ADD PRIMARY KEY (`id`);
--
-- Indexes for table `photo`
--
ALTER TABLE `photo`
ADD PRIMARY KEY (`id`);
--
-- Indexes for table `profile`
--
ALTER TABLE `profile`
ADD PRIMARY KEY (`id`), ADD KEY `name` (`name`), ADD KEY `nurl` (`nurl`), ADD KEY `comm` (`comm`), ADD KEY `pdesc` (`pdesc`), ADD KEY `locality` (`locality`), ADD KEY `region` (`region`), ADD KEY `country-name` (`country-name`), ADD KEY `homepage` (`homepage`), ADD FULLTEXT KEY `tags` (`tags`), ADD FULLTEXT KEY `profile-ft` (`name`,`pdesc`,`homepage`,`locality`,`region`,`country-name`,`tags`);
--
-- Indexes for table `session`
--
ALTER TABLE `session`
ADD PRIMARY KEY (`id`), ADD KEY `sid` (`sid`), ADD KEY `expire` (`expire`);
--
-- Indexes for table `site`
--
ALTER TABLE `site`
ADD PRIMARY KEY (`id`);
--
-- Indexes for table `site-health`
--
ALTER TABLE `site-health`
ADD PRIMARY KEY (`id`), ADD KEY `base_url` (`base_url`), ADD KEY `health_score` (`health_score`), ADD KEY `dt_last_seen` (`dt_last_seen`);
--
-- Indexes for table `site-probe`
--
ALTER TABLE `site-probe`
ADD PRIMARY KEY (`id`), ADD KEY `site_health_id` (`site_health_id`), ADD KEY `dt_performed` (`dt_performed`);
--
-- Indexes for table `site-scrape`
--
ALTER TABLE `site-scrape`
ADD PRIMARY KEY (`id`), ADD KEY `site_health_id` (`site_health_id`), ADD KEY `dt_performed` (`dt_performed`);
--
-- Indexes for table `sync-pull-queue`
--
ALTER TABLE `sync-pull-queue`
ADD PRIMARY KEY (`url`);
--
-- Indexes for table `sync-push-queue`
--
ALTER TABLE `sync-push-queue`
ADD PRIMARY KEY (`url`);
--
-- Indexes for table `sync-targets`
--
ALTER TABLE `sync-targets`
ADD PRIMARY KEY (`base_url`), ADD KEY `push` (`push`), ADD KEY `pull` (`pull`);
--
-- Indexes for table `sync-timestamps`
--
ALTER TABLE `sync-timestamps`
ADD PRIMARY KEY (`url`), ADD KEY `modified` (`modified`);
--
-- Indexes for table `tag`
--
ALTER TABLE `tag`
ADD PRIMARY KEY (`id`), ADD KEY `term` (`term`), ADD KEY `nurl` (`nurl`);
--
-- Indexes for table `user`
--
ALTER TABLE `user`
ADD PRIMARY KEY (`uid`);
--
-- AUTO_INCREMENT for dumped tables
--
--
-- AUTO_INCREMENT for table `flag`
--
ALTER TABLE `flag`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6;
--
-- AUTO_INCREMENT for table `photo`
--
ALTER TABLE `photo`
MODIFY `id` int(10) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=1486;
--
-- AUTO_INCREMENT for table `profile`
--
ALTER TABLE `profile`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=1488;
--
-- AUTO_INCREMENT for table `session`
--
ALTER TABLE `session`
MODIFY `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=104;
--
-- AUTO_INCREMENT for table `site`
--
ALTER TABLE `site`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
--
-- AUTO_INCREMENT for table `site-health`
--
ALTER TABLE `site-health`
MODIFY `id` int(10) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=1513;
--
-- AUTO_INCREMENT for table `site-probe`
--
ALTER TABLE `site-probe`
MODIFY `id` int(10) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6651;
--
-- AUTO_INCREMENT for table `site-scrape`
--
ALTER TABLE `site-scrape`
MODIFY `id` int(10) unsigned NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=21048;
--
-- AUTO_INCREMENT for table `tag`
--
ALTER TABLE `tag`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=2322;
--
-- AUTO_INCREMENT for table `user`
--
ALTER TABLE `user`
MODIFY `uid` int(11) NOT NULL AUTO_INCREMENT;

View File

@ -48,10 +48,9 @@ function directory_content(App $a)
));
if ($search) {
$search = dbesc($search . '*');
$search = dbesc($search);
}
$sql_extra = ((strlen($search)) ? " AND MATCH (`name`, `pdesc`, `homepage`, `locality`, `region`, `country-name`, `tags`)
AGAINST ('$search' IN BOOLEAN MODE) " : "");
$sql_extra = ((strlen($search)) ? " AND MATCH (`name`, `pdesc`, `homepage`, `locality`, `region`, `country-name`, `tags`) AGAINST ('$search') " : "");
if ($forums) {
$sql_extra .= " AND `comm` = 1 ";