diff --git a/boot.php b/boot.php index dd816270c..d7b8268b7 100644 --- a/boot.php +++ b/boot.php @@ -38,7 +38,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Asparagus'); define ( 'FRIENDICA_VERSION', '3.5.1-dev' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1211 ); +define ( 'DB_UPDATE_VERSION', 1212 ); /** * @brief Constant with a HTML line break. diff --git a/database.sql b/database.sql index 2e83c33a6..becd14fc7 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 3.5.1-dev (Asparagus) --- DB_UPDATE_VERSION 1211 +-- DB_UPDATE_VERSION 1212 -- ------------------------------------------ @@ -174,9 +174,16 @@ CREATE TABLE IF NOT EXISTS `contact` ( `fetch_further_information` tinyint(1) NOT NULL DEFAULT 0, `ffi_keyword_blacklist` mediumtext, PRIMARY KEY(`id`), - INDEX `uid` (`uid`), - INDEX `addr_uid` (`addr`,`uid`), - INDEX `nurl` (`nurl`) + INDEX `uid_name` (`uid`,`name`), + INDEX `uid_self` (`uid`,`self`), + INDEX `alias_uid` (`alias`(32),`uid`), + INDEX `uid_pending` (`uid`,`pending`), + INDEX `uid_blocked` (`uid`,`blocked`), + INDEX `uid_rel_network_poll` (`uid`,`rel`,`network`,`poll`(64),`archive`), + INDEX `uid_network_batch` (`uid`,`network`,`batch`(64)), + INDEX `addr_uid` (`addr`(32),`uid`), + INDEX `nurl_uid` (`nurl`(32),`uid`), + INDEX `nick_uid` (`nick`(32),`uid`) ) DEFAULT CHARSET=utf8mb4; -- @@ -232,7 +239,7 @@ CREATE TABLE IF NOT EXISTS `event` ( `deny_cid` mediumtext, `deny_gid` mediumtext, PRIMARY KEY(`id`), - INDEX `uid` (`uid`) + INDEX `uid_start` (`uid`,`start`) ) DEFAULT CHARSET=utf8mb4; -- @@ -257,7 +264,7 @@ CREATE TABLE IF NOT EXISTS `fcontact` ( `pubkey` text, `updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`id`), - INDEX `addr` (`addr`) + INDEX `addr` (`addr`(32)) ) DEFAULT CHARSET=utf8mb4; -- @@ -280,7 +287,7 @@ CREATE TABLE IF NOT EXISTS `fserver` ( `posturl` varchar(255) NOT NULL DEFAULT '', `key` text, PRIMARY KEY(`id`), - INDEX `server` (`server`) + INDEX `server` (`server`(32)) ) DEFAULT CHARSET=utf8mb4; -- @@ -342,10 +349,10 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( `generation` tinyint(3) NOT NULL DEFAULT 0, `server_url` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY(`id`), - INDEX `nurl` (`nurl`), - INDEX `name` (`name`), - INDEX `nick` (`nick`), - INDEX `addr` (`addr`), + INDEX `nurl` (`nurl`(32)), + INDEX `name` (`name`(32)), + INDEX `nick` (`nick`(32)), + INDEX `addr` (`addr`(32)), INDEX `updated` (`updated`) ) DEFAULT CHARSET=utf8mb4; @@ -360,7 +367,7 @@ CREATE TABLE IF NOT EXISTS `glink` ( `zcid` int(11) NOT NULL DEFAULT 0, `updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`id`), - INDEX `cid_uid_gcid_zcid` (`cid`,`uid`,`gcid`,`zcid`), + UNIQUE INDEX `cid_uid_gcid_zcid` (`cid`,`uid`,`gcid`,`zcid`), INDEX `gcid` (`gcid`), INDEX `zcid` (`zcid`) ) DEFAULT CHARSET=utf8mb4; @@ -387,7 +394,9 @@ CREATE TABLE IF NOT EXISTS `group_member` ( `gid` int(10) unsigned NOT NULL DEFAULT 0, `contact-id` int(10) unsigned NOT NULL DEFAULT 0, PRIMARY KEY(`id`), - INDEX `uid_gid_contactid` (`uid`,`gid`,`contact-id`) + INDEX `cid_contactid` (`cid`,`contact-id`), + INDEX `uid_contactid` (`uid`,`contact-id`), + UNIQUE INDEX `uid_gid_contactid` (`uid`,`gid`,`contact-id`) ) DEFAULT CHARSET=utf8mb4; -- @@ -410,7 +419,7 @@ CREATE TABLE IF NOT EXISTS `gserver` ( `last_contact` datetime DEFAULT '0000-00-00 00:00:00', `last_failure` datetime DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`id`), - INDEX `nurl` (`nurl`) + INDEX `nurl` (`nurl`(32)) ) DEFAULT CHARSET=utf8mb4; -- @@ -525,24 +534,17 @@ CREATE TABLE IF NOT EXISTS `item` ( INDEX `uid_network_received` (`uid`,`network`,`received`), INDEX `uid_received` (`uid`,`received`), INDEX `uid_network_commented` (`uid`,`network`,`commented`), - INDEX `uid_commented` (`uid`,`commented`), INDEX `uid_title` (`uid`,`title`), INDEX `uid_thrparent` (`uid`,`thr-parent`), INDEX `uid_parenturi` (`uid`,`parent-uri`), INDEX `uid_contactid_id` (`uid`,`contact-id`,`id`), INDEX `uid_contactid_created` (`uid`,`contact-id`,`created`), - INDEX `gcontactid_uid_created` (`gcontact-id`,`uid`,`created`), INDEX `authorid_created` (`author-id`,`created`), - INDEX `ownerid_created` (`owner-id`,`created`), - INDEX `wall_body` (`wall`,`body`(6)), - INDEX `uid_visible_moderated_created` (`uid`,`visible`,`moderated`,`created`), INDEX `uid_uri` (`uid`,`uri`), INDEX `uid_wall_created` (`uid`,`wall`,`created`), INDEX `resource-id` (`resource-id`), INDEX `uid_type` (`uid`,`type`), - INDEX `uid_starred_id` (`uid`,`starred`,`id`), INDEX `contactid_allowcid_allowpid_denycid_denygid` (`contact-id`,`allow_cid`(10),`allow_gid`(10),`deny_cid`(10),`deny_gid`(10)), - INDEX `uid_wall_parent_created` (`uid`,`wall`,`parent`,`created`), INDEX `uid_type_changed` (`uid`,`type`,`changed`), INDEX `contactid_verb` (`contact-id`,`verb`), INDEX `deleted_changed` (`deleted`,`changed`), @@ -564,7 +566,7 @@ CREATE TABLE IF NOT EXISTS `item_id` ( PRIMARY KEY(`id`), INDEX `uid` (`uid`), INDEX `sid` (`sid`), - INDEX `service` (`service`), + INDEX `service` (`service`(32)), INDEX `iid` (`iid`) ) DEFAULT CHARSET=utf8mb4; @@ -602,11 +604,10 @@ CREATE TABLE IF NOT EXISTS `mail` ( `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`id`), INDEX `uid` (`uid`), - INDEX `guid` (`guid`), + INDEX `uid_seen` (`uid`,`seen`), INDEX `convid` (`convid`), - INDEX `reply` (`reply`), - INDEX `uri` (`uri`), - INDEX `parent-uri` (`parent-uri`) + INDEX `uri` (`uri`(64)), + INDEX `parent-uri` (`parent-uri`(64)) ) DEFAULT CHARSET=utf8mb4; -- @@ -662,7 +663,11 @@ CREATE TABLE IF NOT EXISTS `notify` ( `name_cache` tinytext, `msg_cache` mediumtext, PRIMARY KEY(`id`), - INDEX `uid` (`uid`) + INDEX `uid_hash` (`uid`,`hash`), + INDEX `uid_seen_date` (`uid`,`seen`,`date`), + INDEX `uid_type_link` (`uid`,`type`,`link`), + INDEX `uid_link` (`uid`,`link`), + INDEX `uid_date` (`uid`,`date`) ) DEFAULT CHARSET=utf8mb4; -- @@ -675,8 +680,7 @@ CREATE TABLE IF NOT EXISTS `notify-threads` ( `parent-item` int(10) unsigned NOT NULL DEFAULT 0, `receiver-uid` int(11) NOT NULL DEFAULT 0, PRIMARY KEY(`id`), - INDEX `master-parent-item` (`master-parent-item`), - INDEX `receiver-uid` (`receiver-uid`) + INDEX `master-parent-item` (`master-parent-item`) ) DEFAULT CHARSET=utf8mb4; -- @@ -745,9 +749,9 @@ CREATE TABLE IF NOT EXISTS `photo` ( PRIMARY KEY(`id`), INDEX `uid_contactid` (`uid`,`contact-id`), INDEX `uid_profile` (`uid`,`profile`), - INDEX `uid_album_created` (`uid`,`album`,`created`), - INDEX `resource-id` (`resource-id`), - INDEX `guid` (`guid`) + INDEX `uid_album_created` (`uid`,`album`(32),`created`), + INDEX `uid_album_resource-id_created` (`uid`,`album`(32),`resource-id`(64),`created`), + INDEX `resource-id` (`resource-id`(64)) ) DEFAULT CHARSET=utf8mb4; -- @@ -839,8 +843,7 @@ CREATE TABLE IF NOT EXISTS `profile` ( `thumb` varchar(255) NOT NULL DEFAULT '', `publish` tinyint(1) NOT NULL DEFAULT 0, `net-publish` tinyint(1) NOT NULL DEFAULT 0, - PRIMARY KEY(`id`), - INDEX `hometown` (`hometown`) + PRIMARY KEY(`id`) ) DEFAULT CHARSET=utf8mb4; -- @@ -912,8 +915,7 @@ CREATE TABLE IF NOT EXISTS `search` ( `uid` int(11) NOT NULL DEFAULT 0, `term` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY(`id`), - INDEX `uid` (`uid`), - INDEX `term` (`term`) + INDEX `uid` (`uid`) ) DEFAULT CHARSET=utf8mb4; -- @@ -925,7 +927,7 @@ CREATE TABLE IF NOT EXISTS `session` ( `data` text, `expire` int(10) unsigned NOT NULL DEFAULT 0, PRIMARY KEY(`id`), - INDEX `sid` (`sid`), + INDEX `sid` (`sid`(64)), INDEX `expire` (`expire`) ) DEFAULT CHARSET=utf8mb4; @@ -977,12 +979,11 @@ CREATE TABLE IF NOT EXISTS `term` ( `uid` int(10) unsigned NOT NULL DEFAULT 0, PRIMARY KEY(`tid`), INDEX `oid_otype_type_term` (`oid`,`otype`,`type`,`term`), - INDEX `uid_term_tid` (`uid`,`term`,`tid`), - INDEX `type_term` (`type`,`term`), - INDEX `uid_otype_type_term_global_created` (`uid`,`otype`,`type`,`term`,`global`,`created`), - INDEX `otype_type_term_tid` (`otype`,`type`,`term`,`tid`), - INDEX `uid_otype_type_url` (`uid`,`otype`,`type`,`url`), - INDEX `guid` (`guid`) + INDEX `uid_term_tid` (`uid`,`term`(32),`tid`), + INDEX `type_term` (`type`,`term`(32)), + INDEX `uid_otype_type_term_global_created` (`uid`,`otype`,`type`,`term`(32),`global`,`created`), + INDEX `uid_otype_type_url` (`uid`,`otype`,`type`,`url`(64)), + INDEX `guid` (`guid`(64)) ) DEFAULT CHARSET=utf8mb4; -- @@ -1022,9 +1023,6 @@ CREATE TABLE IF NOT EXISTS `thread` ( INDEX `uid_network_created` (`uid`,`network`,`created`), INDEX `uid_contactid_commented` (`uid`,`contact-id`,`commented`), INDEX `uid_contactid_created` (`uid`,`contact-id`,`created`), - INDEX `uid_gcontactid_commented` (`uid`,`gcontact-id`,`commented`), - INDEX `uid_gcontactid_created` (`uid`,`gcontact-id`,`created`), - INDEX `wall_private_received` (`wall`,`private`,`received`), INDEX `uid_created` (`uid`,`created`), INDEX `uid_commented` (`uid`,`commented`) ) DEFAULT CHARSET=utf8mb4; @@ -1090,7 +1088,7 @@ CREATE TABLE IF NOT EXISTS `user` ( `deny_gid` mediumtext, `openidserver` text, PRIMARY KEY(`uid`), - INDEX `nickname` (`nickname`) + INDEX `nickname` (`nickname`(32)) ) DEFAULT CHARSET=utf8mb4; -- @@ -1100,7 +1098,7 @@ CREATE TABLE IF NOT EXISTS `userd` ( `id` int(11) NOT NULL auto_increment, `username` varchar(255) NOT NULL, PRIMARY KEY(`id`), - INDEX `username` (`username`) + INDEX `username` (`username`(32)) ) DEFAULT CHARSET=utf8mb4; -- diff --git a/doc/htconfig.md b/doc/htconfig.md index fb257c3da..05a2a7a96 100644 --- a/doc/htconfig.md +++ b/doc/htconfig.md @@ -25,6 +25,7 @@ Example: To set the directory value please add this line to your .htconfig.php: * **allowed_link_protocols** (Array) - Allowed protocols in links URLs, add at your own risk. http is always allowed. * **birthday_input_format** - Default value is "ymd". * **block_local_dir** (Boolean) - Blocks the access to the directory of the local users. +* **dbclean** (Boolean) - Enable the automatic database cleanup process * **default_service_class** - * **delivery_batch_count** - Number of deliveries per process. Default value is 1. (Disabled when using the worker) * **diaspora_test** (Boolean) - For development only. Disables the message transfer. @@ -45,7 +46,8 @@ Example: To set the directory value please add this line to your .htconfig.php: * **max_processes_frontend** - Maximum number of concurrent database processes for foreground tasks. Default value is 20. * **memcache** (Boolean) - Use memcache. To use memcache the PECL extension "memcache" has to be installed and activated. * **memcache_host** - Hostname of the memcache daemon. Default is '127.0.0.1'. -* **memcache_port** - Portnumberof the memcache daemon. Default is 11211. +* **memcache_port** - Portnumber of the memcache daemon. Default is 11211. +* **no_count** (Boolean) - Don't do count calculations (currently only when showing albums) * **no_oembed** (Boolean) - Don't use OEmbed to fetch more information about a link. * **no_oembed_rich_content** (Boolean) - Don't show the rich content (e.g. embedded PDF). * **no_smilies** (Boolean) - Don't show smilies. diff --git a/include/Contact.php b/include/Contact.php index 5d8ccc452..9f0dd45c5 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -636,7 +636,7 @@ function posts_from_gcontact($a, $gcontact_id) { $r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, `author-name` AS `name`, `owner-avatar` AS `photo`, `owner-link` AS `url`, `owner-avatar` AS `thumb` - FROM `item` FORCE INDEX (`gcontactid_uid_created`) + FROM `item` WHERE `gcontact-id` = %d AND $sql AND NOT `deleted` AND NOT `moderated` AND `visible` ORDER BY `item`.`created` DESC LIMIT %d, %d", diff --git a/include/acl_selectors.php b/include/acl_selectors.php index f6c4f947e..75a4985c8 100644 --- a/include/acl_selectors.php +++ b/include/acl_selectors.php @@ -135,7 +135,7 @@ function contact_selector($selname, $selclass, $preselected = false, $options) { $o .= "\r\n"; $r = q("SELECT `id`, `name`, `url`, `network` FROM `contact` - WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 AND `notify` != '' + WHERE `uid` = %d AND NOT `self` AND NOT `blocked` AND NOT `pending` AND NOT `archive` AND `notify` != '' $sql_extra ORDER BY `name` ASC ", intval(local_user()) @@ -449,8 +449,8 @@ function acl_lookup(&$a, $out_type = 'json') { // autocomplete for editor mentions if ($type=='' || $type=='c'){ $r = q("SELECT COUNT(*) AS c FROM `contact` - WHERE `uid` = %d AND `self` = 0 - AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 + WHERE `uid` = %d AND NOT `self` + AND NOT `blocked` AND NOT `pending` AND NOT `archive` AND `notify` != '' $sql_extra2" , intval(local_user()) ); @@ -461,8 +461,8 @@ function acl_lookup(&$a, $out_type = 'json') { // autocomplete for Private Messages $r = q("SELECT COUNT(*) AS c FROM `contact` - WHERE `uid` = %d AND `self` = 0 - AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 + WHERE `uid` = %d AND NOT `self` + AND NOT `blocked` AND NOT `pending` AND NOT `archive` AND `network` IN ('%s','%s','%s') $sql_extra2" , intval(local_user()), dbesc(NETWORK_DFRN), @@ -477,8 +477,8 @@ function acl_lookup(&$a, $out_type = 'json') { // autocomplete for Contacts $r = q("SELECT COUNT(*) AS c FROM `contact` - WHERE `uid` = %d AND `self` = 0 - AND `pending` = 0 $sql_extra2" , + WHERE `uid` = %d AND NOT `self` + AND NOT `pending` $sql_extra2" , intval(local_user()) ); $contact_count = (int)$r[0]['c']; @@ -525,7 +525,7 @@ function acl_lookup(&$a, $out_type = 'json') { if ($type==''){ $r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag`, `forum`, `prv` FROM `contact` - WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 AND `notify` != '' + WHERE `uid` = %d AND NOT `self` AND NOT `blocked` AND NOT `pending` AND NOT `archive` AND `notify` != '' AND NOT (`network` IN ('%s', '%s')) $sql_extra2 ORDER BY `name` ASC ", @@ -536,7 +536,7 @@ function acl_lookup(&$a, $out_type = 'json') { elseif ($type=='c'){ $r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag`, `forum`, `prv` FROM `contact` - WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 AND `notify` != '' + WHERE `uid` = %d AND NOT `self` AND NOT `blocked` AND NOT `pending` AND NOT `archive` AND `notify` != '' AND NOT (`network` IN ('%s')) $sql_extra2 ORDER BY `name` ASC ", @@ -546,7 +546,7 @@ function acl_lookup(&$a, $out_type = 'json') { } elseif($type == 'm') { $r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag` FROM `contact` - WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 + WHERE `uid` = %d AND NOT `self` AND NOT `blocked` AND NOT `pending` AND NOT `archive` AND `network` IN ('%s','%s','%s') $sql_extra2 ORDER BY `name` ASC ", diff --git a/include/cron.php b/include/cron.php index e98239b82..97744ed27 100644 --- a/include/cron.php +++ b/include/cron.php @@ -128,7 +128,7 @@ function cron_run(&$argv, &$argc){ proc_run(PRIORITY_LOW, 'include/expire.php'); - proc_run(PRIORITY_LOW, 'include/dbclean.php'); + proc_run(PRIORITY_MEDIUM, 'include/dbclean.php'); cron_update_photo_albums(); } diff --git a/include/dbclean.php b/include/dbclean.php index 0baece816..c80e5a3be 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -3,13 +3,18 @@ * @file include/dbclean.php * @brief The script is called from time to time to clean the database entries and remove orphaned data. */ + +use \Friendica\Core\Config; +use \Friendica\Core\PConfig; + require_once("boot.php"); function dbclean_run(&$argv, &$argc) { global $a, $db; - if (is_null($a)) + if (is_null($a)) { $a = new App; + } if (is_null($db)) { @include(".htconfig.php"); @@ -18,8 +23,12 @@ function dbclean_run(&$argv, &$argc) { unset($db_host, $db_user, $db_pass, $db_data); } - load_config('config'); - load_config('system'); + Config::load('config'); + Config::load('system'); + + if (!Config::get('system', 'dbclean', false)) { + return; + } if ($argc == 2) { $stage = intval($argv[1]); @@ -27,7 +36,7 @@ function dbclean_run(&$argv, &$argc) { $stage = 0; } - if (get_config("system", "worker") AND ($stage == 0)) { + if (Config::get("system", "worker") AND ($stage == 0)) { proc_run(PRIORITY_LOW, 'include/dbclean.php', 1); proc_run(PRIORITY_LOW, 'include/dbclean.php', 2); proc_run(PRIORITY_LOW, 'include/dbclean.php', 3); @@ -48,11 +57,18 @@ function remove_orphans($stage = 0) { $count = 0; + // With activated worker we split the deletion in many small tasks + if (Config::get("system", "worker")) { + $limit = 1000; + } else { + $limit = 10000; + } + if (($stage == 1) OR ($stage == 0)) { logger("Deleting old global item entries from item table without user copy"); if ($db->q("SELECT `id` FROM `item` WHERE `uid` = 0 AND NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) - AND `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY LIMIT 10000", true)) { + AND `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY LIMIT ".intval($limit), true)) { $count = $db->num_rows(); logger("found global item orphans: ".$count); while ($orphan = $db->qfetch()) { @@ -65,7 +81,7 @@ function remove_orphans($stage = 0) { if (($stage == 2) OR ($stage == 0)) { logger("Deleting items without parents"); - if ($db->q("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT 10000", true)) { + if ($db->q("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit), true)) { $count = $db->num_rows(); logger("found item orphans without parents: ".$count); while ($orphan = $db->qfetch()) { @@ -78,7 +94,7 @@ function remove_orphans($stage = 0) { if (($stage == 3) OR ($stage == 0)) { logger("Deleting orphaned data from thread table"); - if ($db->q("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`)", true)) { + if ($db->q("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit), true)) { $count = $db->num_rows(); logger("found thread orphans: ".$count); while ($orphan = $db->qfetch()) { @@ -91,7 +107,7 @@ function remove_orphans($stage = 0) { if (($stage == 4) OR ($stage == 0)) { logger("Deleting orphaned data from notify table"); - if ($db->q("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`)", true)) { + if ($db->q("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit), true)) { $count = $db->num_rows(); logger("found notify orphans: ".$count); while ($orphan = $db->qfetch()) { @@ -104,7 +120,7 @@ function remove_orphans($stage = 0) { if (($stage == 5) OR ($stage == 0)) { logger("Deleting orphaned data from notify-threads table"); - if ($db->q("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`)", true)) { + if ($db->q("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit), true)) { $count = $db->num_rows(); logger("found notify-threads orphans: ".$count); while ($orphan = $db->qfetch()) { @@ -118,7 +134,7 @@ function remove_orphans($stage = 0) { if (($stage == 6) OR ($stage == 0)) { logger("Deleting orphaned data from sign table"); - if ($db->q("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`)", true)) { + if ($db->q("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit), true)) { $count = $db->num_rows(); logger("found sign orphans: ".$count); while ($orphan = $db->qfetch()) { @@ -132,7 +148,7 @@ function remove_orphans($stage = 0) { if (($stage == 7) OR ($stage == 0)) { logger("Deleting orphaned data from term table"); - if ($db->q("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`)", true)) { + if ($db->q("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit), true)) { $count = $db->num_rows(); logger("found term orphans: ".$count); while ($orphan = $db->qfetch()) { @@ -144,8 +160,8 @@ function remove_orphans($stage = 0) { } // Call it again if not all entries were purged - if (($stage != 0) AND ($count > 0) AND get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php'); + if (($stage != 0) AND ($count > 0) AND Config::get("system", "worker")) { + proc_run(PRIORITY_MEDIUM, 'include/dbclean.php'); } } diff --git a/include/dbstructure.php b/include/dbstructure.php index 4a8bac198..5c89f49fb 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -586,9 +586,16 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "uid" => array("uid"), - "addr_uid" => array("addr", "uid"), - "nurl" => array("nurl"), + "uid_name" => array("uid", "name"), + "uid_self" => array("uid", "self"), + "alias_uid" => array("alias(32)", "uid"), + "uid_pending" => array("uid", "pending"), + "uid_blocked" => array("uid", "blocked"), + "uid_rel_network_poll" => array("uid", "rel", "network", "poll(64)", "archive"), + "uid_network_batch" => array("uid", "network", "batch(64)"), + "addr_uid" => array("addr(32)", "uid"), + "nurl_uid" => array("nurl(32)", "uid"), + "nick_uid" => array("nick(32)", "uid"), ) ); $database["conv"] = array( @@ -644,7 +651,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "uid" => array("uid"), + "uid_start" => array("uid", "start"), ) ); $database["fcontact"] = array( @@ -669,7 +676,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "addr" => array("addr"), + "addr" => array("addr(32)"), ) ); $database["ffinder"] = array( @@ -692,7 +699,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "server" => array("server"), + "server" => array("server(32)"), ) ); $database["fsuggest"] = array( @@ -754,10 +761,10 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "nurl" => array("nurl"), - "name" => array("name"), - "nick" => array("nick"), - "addr" => array("addr"), + "nurl" => array("nurl(32)"), + "name" => array("name(32)"), + "nick" => array("nick(32)"), + "addr" => array("addr(32)"), "updated" => array("updated"), ) ); @@ -772,7 +779,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "cid_uid_gcid_zcid" => array("cid","uid","gcid","zcid"), + "cid_uid_gcid_zcid" => array("UNIQUE", "cid","uid","gcid","zcid"), "gcid" => array("gcid"), "zcid" => array("zcid"), ) @@ -799,7 +806,9 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "uid_gid_contactid" => array("uid","gid","contact-id"), + "gid_contactid" => array("gid", "contact-id"), + "uid_contactid" => array("uid", "contact-id"), + "uid_gid_contactid" => array("UNIQUE", "uid", "gid", "contact-id"), ) ); $database["gserver"] = array( @@ -822,7 +831,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "nurl" => array("nurl"), + "nurl" => array("nurl(32)"), ) ); $database["hook"] = array( @@ -937,24 +946,17 @@ function db_definition($charset) { "uid_network_received" => array("uid","network","received"), "uid_received" => array("uid","received"), "uid_network_commented" => array("uid","network","commented"), - "uid_commented" => array("uid","commented"), "uid_title" => array("uid","title"), "uid_thrparent" => array("uid","thr-parent"), "uid_parenturi" => array("uid","parent-uri"), "uid_contactid_id" => array("uid","contact-id","id"), "uid_contactid_created" => array("uid","contact-id","created"), - "gcontactid_uid_created" => array("gcontact-id","uid","created"), "authorid_created" => array("author-id","created"), - "ownerid_created" => array("owner-id","created"), - "wall_body" => array("wall","body(6)"), - "uid_visible_moderated_created" => array("uid","visible","moderated","created"), "uid_uri" => array("uid", "uri"), "uid_wall_created" => array("uid","wall","created"), "resource-id" => array("resource-id"), "uid_type" => array("uid","type"), - "uid_starred_id" => array("uid","starred", "id"), "contactid_allowcid_allowpid_denycid_denygid" => array("contact-id","allow_cid(10)","allow_gid(10)","deny_cid(10)","deny_gid(10)"), - "uid_wall_parent_created" => array("uid","wall","parent","created"), "uid_type_changed" => array("uid","type","changed"), "contactid_verb" => array("contact-id","verb"), "deleted_changed" => array("deleted","changed"), @@ -976,7 +978,7 @@ function db_definition($charset) { "PRIMARY" => array("id"), "uid" => array("uid"), "sid" => array("sid"), - "service" => array("service"), + "service" => array("service(32)"), "iid" => array("iid"), ) ); @@ -1014,11 +1016,10 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("id"), "uid" => array("uid"), - "guid" => array("guid"), + "uid_seen" => array("uid", "seen"), "convid" => array("convid"), - "reply" => array("reply"), - "uri" => array("uri"), - "parent-uri" => array("parent-uri"), + "uri" => array("uri(64)"), + "parent-uri" => array("parent-uri(64)"), ) ); $database["mailacct"] = array( @@ -1074,7 +1075,11 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "uid" => array("uid"), + "uid_hash" => array("uid", "hash"), + "uid_seen_date" => array("uid", "seen", "date"), + "uid_type_link" => array("uid", "type", "link"), + "uid_link" => array("uid", "link"), + "uid_date" => array("uid", "date"), ) ); $database["notify-threads"] = array( @@ -1088,7 +1093,6 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("id"), "master-parent-item" => array("master-parent-item"), - "receiver-uid" => array("receiver-uid"), ) ); $database["oembed"] = array( @@ -1157,9 +1161,9 @@ function db_definition($charset) { "PRIMARY" => array("id"), "uid_contactid" => array("uid", "contact-id"), "uid_profile" => array("uid", "profile"), - "uid_album_created" => array("uid", "album", "created"), - "resource-id" => array("resource-id"), - "guid" => array("guid"), + "uid_album_created" => array("uid", "album(32)", "created"), + "uid_album_resource-id_created" => array("uid", "album(32)", "resource-id(64)", "created"), + "resource-id" => array("resource-id(64)"), ) ); $database["poll"] = array( @@ -1252,7 +1256,6 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "hometown" => array("hometown"), ) ); $database["profile_check"] = array( @@ -1325,7 +1328,6 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("id"), "uid" => array("uid"), - "term" => array("term"), ) ); $database["session"] = array( @@ -1337,7 +1339,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "sid" => array("sid"), + "sid" => array("sid(64)"), "expire" => array("expire"), ) ); @@ -1389,12 +1391,11 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("tid"), "oid_otype_type_term" => array("oid","otype","type","term"), - "uid_term_tid" => array("uid","term","tid"), - "type_term" => array("type","term"), - "uid_otype_type_term_global_created" => array("uid","otype","type","term","global","created"), - "otype_type_term_tid" => array("otype","type","term","tid"), - "uid_otype_type_url" => array("uid","otype","type","url"), - "guid" => array("guid"), + "uid_term_tid" => array("uid","term(32)","tid"), + "type_term" => array("type","term(32)"), + "uid_otype_type_term_global_created" => array("uid","otype","type","term(32)","global","created"), + "uid_otype_type_url" => array("uid","otype","type","url(64)"), + "guid" => array("guid(64)"), ) ); $database["thread"] = array( @@ -1434,9 +1435,6 @@ function db_definition($charset) { "uid_network_created" => array("uid","network","created"), "uid_contactid_commented" => array("uid","contact-id","commented"), "uid_contactid_created" => array("uid","contact-id","created"), - "uid_gcontactid_commented" => array("uid","gcontact-id","commented"), - "uid_gcontactid_created" => array("uid","gcontact-id","created"), - "wall_private_received" => array("wall","private","received"), "uid_created" => array("uid","created"), "uid_commented" => array("uid","commented"), ) @@ -1502,7 +1500,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("uid"), - "nickname" => array("nickname"), + "nickname" => array("nickname(32)"), ) ); $database["userd"] = array( @@ -1512,7 +1510,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "username" => array("username"), + "username" => array("username(32)"), ) ); $database["workerqueue"] = array( diff --git a/include/diaspora.php b/include/diaspora.php index db2cca471..fde086e20 100644 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -736,13 +736,28 @@ class Diaspora { * @return The contact id */ private static function contact_by_handle($uid, $handle) { + + // First do a direct search on the contact table $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `addr` = '%s' LIMIT 1", intval($uid), dbesc($handle) ); - if ($r) + if (dbm::is_result($r)) { return $r[0]; + } else { + // We haven't found it? + // We use another function for it that will possibly create a contact entry + $cid = get_contact($handle, $uid); + + if ($cid > 0) { + $r = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1", intval($cid)); + + if (dbm::is_result($r)) { + return $r[0]; + } + } + } $handle_parts = explode("@", $handle); $nurl_sql = "%%://".$handle_parts[1]."%%/profile/".$handle_parts[0]; @@ -751,9 +766,11 @@ class Diaspora { intval($uid), dbesc($nurl_sql) ); - if($r) + if (dbm::is_result($r)) { return $r[0]; + } + logger("Haven't found contact for user ".$uid." and handle ".$handle, LOGGER_DEBUG); return false; } @@ -833,7 +850,7 @@ class Diaspora { dbesc($guid) ); - if ($r) { + if (dbm::is_result($r)) { logger("message ".$guid." already exists for user ".$uid); return $r[0]["id"]; } diff --git a/include/enotify.php b/include/enotify.php index aa29e117a..ebc27309d 100644 --- a/include/enotify.php +++ b/include/enotify.php @@ -105,7 +105,7 @@ function notification($params) { // If so don't create a second notification $p = null; - $p = q("SELECT `id` FROM `notify` WHERE (`type` = %d OR `type` = %d OR `type` = %d) AND `link` = '%s' AND `uid` = %d LIMIT 1", + $p = q("SELECT `id` FROM `notify` WHERE `type` IN (%d, %d, %d) AND `link` = '%s' AND `uid` = %d LIMIT 1", intval(NOTIFY_TAGSELF), intval(NOTIFY_COMMENT), intval(NOTIFY_SHARE), @@ -472,7 +472,7 @@ function notification($params) { // After we've stored everything, look again to see if there are any duplicates and if so remove them $p = null; - $p = q("SELECT `id` FROM `notify` WHERE (`type` = %d OR `type` = %d) AND `link` = '%s' AND `uid` = %d ORDER BY `id`", + $p = q("SELECT `id` FROM `notify` WHERE `type` IN (%d, %d) AND `link` = '%s' AND `uid` = %d ORDER BY `id`", intval(NOTIFY_TAGSELF), intval(NOTIFY_COMMENT), dbesc($params['link']), diff --git a/include/photos.php b/include/photos.php index 2beb23ffa..7cdd14bf6 100644 --- a/include/photos.php +++ b/include/photos.php @@ -4,6 +4,9 @@ * @brief Functions related to photo handling. */ +use \Friendica\Core\Config; +use \Friendica\Core\PConfig; + function getGps($exifCoord, $hemi) { $degrees = count($exifCoord) > 0 ? gps2Num($exifCoord[0]) : 0; $minutes = count($exifCoord) > 1 ? gps2Num($exifCoord[1]) : 0; @@ -42,16 +45,28 @@ function photo_albums($uid, $update = false) { $key = "photo_albums:".$uid.":".local_user().":".remote_user(); $albums = Cache::get($key); if (is_null($albums) OR $update) { - /// @todo This query needs to be renewed. It is really slow - // At this time we just store the data in the cache - $albums = qu("SELECT COUNT(DISTINCT `resource-id`) AS `total`, `album` - FROM `photo` USE INDEX (`uid_album_created`) - WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra - GROUP BY `album` ORDER BY `created` DESC", - intval($uid), - dbesc('Contact Photos'), - dbesc(t('Contact Photos')) - ); + if (!Config::get('system', 'no_count', false)) { + /// @todo This query needs to be renewed. It is really slow + // At this time we just store the data in the cache + $albums = qu("SELECT COUNT(DISTINCT `resource-id`) AS `total`, `album` + FROM `photo` USE INDEX (`uid_album_created`) + WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra + GROUP BY `album` ORDER BY `created` DESC", + intval($uid), + dbesc('Contact Photos'), + dbesc(t('Contact Photos')) + ); + } else { + // This query doesn't do the count and is much faster + $albums = qu("SELECT DISTINCT(`album`), '' AS `total` + FROM `photo` USE INDEX (`uid_album_created`) + WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra + GROUP BY `album` ORDER BY `created` DESC", + intval($uid), + dbesc('Contact Photos'), + dbesc(t('Contact Photos')) + ); + } Cache::set($key, $albums, CACHE_DAY); } return $albums; diff --git a/include/redir.php b/include/redir.php index d8bb76439..d29159ed0 100644 --- a/include/redir.php +++ b/include/redir.php @@ -27,13 +27,13 @@ function auto_redir(&$a, $contact_nick) { $baseurl = substr($baseurl, $domain_st + 3); $nurl = normalise_link($baseurl); - - $r = q("SELECT id FROM contact WHERE uid = ( SELECT uid FROM user WHERE nickname = '%s' LIMIT 1 ) - AND nick = '%s' AND self = 0 AND ( url LIKE '%%%s%%' or nurl LIKE '%%%s%%' ) AND blocked = 0 AND pending = 0 LIMIT 1", - dbesc($contact_nick), - dbesc($a->user['nickname']), - dbesc($baseurl), - dbesc($nurl) + /// @todo Why is there a query for "url" *and* "nurl"? Especially this normalising is strange. + $r = q("SELECT `id` FROM `contact` WHERE `uid` = (SELECT `uid` FROM `user` WHERE `nickname` = '%s' LIMIT 1) + AND `nick` = '%s' AND NOT `self` AND (`url` LIKE '%%%s%%' OR `nurl` LIKE '%%%s%%') AND NOT `blocked` AND NOT `pending` LIMIT 1", + dbesc($contact_nick), + dbesc($a->user['nickname']), + dbesc($baseurl), + dbesc($nurl) ); if ((! dbm::is_result($r)) || $r[0]['id'] == remote_user()) { diff --git a/mod/community.php b/mod/community.php index 2c2ea32b3..829797c57 100644 --- a/mod/community.php +++ b/mod/community.php @@ -121,7 +121,7 @@ function community_getitems($start, $itemspage) { return(community_getpublicitems($start, $itemspage)); $r = qu("SELECT %s - FROM `thread` FORCE INDEX (`wall_private_received`) + FROM `thread` INNER JOIN `user` ON `user`.`uid` = `thread`.`uid` AND NOT `user`.`hidewall` INNER JOIN `item` ON `item`.`id` = `thread`.`iid` AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' diff --git a/mod/photos.php b/mod/photos.php index b451535e4..2e86f7fbb 100644 --- a/mod/photos.php +++ b/mod/photos.php @@ -182,11 +182,11 @@ function photos_post(App &$a) { return; // NOTREACHED } - $r = qu("SELECT count(*) FROM `photo` WHERE `album` = '%s' AND `uid` = %d", + $r = qu("SELECT `album` FROM `photo` WHERE `album` = '%s' AND `uid` = %d", dbesc($album), intval($page_owner_uid) ); - if (! dbm::is_result($r)) { + if (!dbm::is_result($r)) { notice( t('Album not found.') . EOL); goaway($_SESSION['photo_return']); return; // NOTREACHED @@ -818,21 +818,22 @@ function photos_post(App &$a) { $imagedata = @file_get_contents($src); - - $r = q("select sum(octet_length(data)) as total from photo where uid = %d and scale = 0 and album != 'Contact Photos' ", - intval($a->data['user']['uid']) - ); - $limit = service_class_fetch($a->data['user']['uid'],'photo_upload_limit'); - if (($limit !== false) && (($r[0]['total'] + strlen($imagedata)) > $limit)) { - notice( upgrade_message() . EOL ); - @unlink($src); - $foo = 0; - call_hooks('photo_post_end',$foo); - killme(); - } + if ($limit) { + $r = q("select sum(octet_length(data)) as total from photo where uid = %d and scale = 0 and album != 'Contact Photos' ", + intval($a->data['user']['uid']) + ); + $size = $r[0]['total']; + if (($size + strlen($imagedata)) > $limit) { + notice( upgrade_message() . EOL ); + @unlink($src); + $foo = 0; + call_hooks('photo_post_end',$foo); + killme(); + } + } $ph = new Photo($imagedata, $type); diff --git a/mod/wall_attach.php b/mod/wall_attach.php index 181256854..5a7471782 100644 --- a/mod/wall_attach.php +++ b/mod/wall_attach.php @@ -112,23 +112,25 @@ function wall_attach_post(App &$a) { killme(); } - $r = q("select sum(octet_length(data)) as total from attach where uid = %d ", - intval($page_owner_uid) - ); - $limit = service_class_fetch($page_owner_uid,'attach_upload_limit'); - if(($limit !== false) && (($r[0]['total'] + strlen($imagedata)) > $limit)) { - $msg = upgrade_message(true); - if ($r_json) { - echo json_encode(array('error'=>$msg)); - } else { - echo $msg. EOL ; - } - @unlink($src); - killme(); - } + if ($limit) { + $r = q("select sum(octet_length(data)) as total from photo where uid = %d and scale = 0 and album != 'Contact Photos' ", + intval($page_owner_uid) + ); + $size = $r[0]['total']; + if (($size + strlen($imagedata)) > $limit) { + $msg = upgrade_message(true); + if ($r_json) { + echo json_encode(array('error'=>$msg)); + } else { + echo $msg. EOL ; + } + @unlink($src); + killme(); + } + } $filedata = @file_get_contents($src); $mimetype = z_mime_content_type($filename); diff --git a/mod/wall_upload.php b/mod/wall_upload.php index eb2a92323..c789262c6 100644 --- a/mod/wall_upload.php +++ b/mod/wall_upload.php @@ -166,21 +166,19 @@ function wall_upload_post(&$a, $desktopmode = true) { intval($page_owner_uid) ); $size = $r[0]['total']; - } else - $size = 0; - if(($limit !== false) && (($size + strlen($imagedata)) > $limit)) { - $msg = upgrade_message(true); - if ($r_json) { - echo json_encode(array('error'=>$msg)); - } else { - echo $msg. EOL; + if (($size + strlen($imagedata)) > $limit) { + $msg = upgrade_message(true); + if ($r_json) { + echo json_encode(array('error'=>$msg)); + } else { + echo $msg. EOL; + } + @unlink($src); + killme(); } - @unlink($src); - killme(); } - $imagedata = @file_get_contents($src); $ph = new Photo($imagedata, $filetype); diff --git a/update.php b/update.php index 3bd9cbe61..058536d82 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ user['uid'])); - + $r = q("SELECT `micro` FROM `contact` WHERE `uid` = %d AND `self`", intval($a->user['uid'])); + $r[0]['photo'] = (dbm::is_result($r) ? $a->remove_baseurl($r[0]['micro']) : "images/person-48.jpg"); $r[0]['name'] = $a->user['username']; @@ -298,7 +298,7 @@ function frio_acl_lookup($a, &$results) { $r = q("SELECT COUNT(*) AS `total` FROM `contact` - WHERE `uid` = %d AND `self` = 0 AND `pending` = 0 $sql_extra $sql_extra2 ", + WHERE `uid` = %d AND NOT `self` AND NOT `pending` $sql_extra $sql_extra2 ", intval($_SESSION['uid'])); if (dbm::is_result($r)) { $total = $r[0]["total"]; @@ -306,7 +306,7 @@ function frio_acl_lookup($a, &$results) { $sql_extra3 = unavailable_networks(); - $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `self` = 0 AND `pending` = 0 $sql_extra $sql_extra2 $sql_extra3 ORDER BY `name` ASC LIMIT 100 ", + $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND NOT `self` AND NOT `pending` $sql_extra $sql_extra2 $sql_extra3 ORDER BY `name` ASC LIMIT 100 ", intval($_SESSION['uid']) );