From 0277c55c437c26f1a6ce87eeffd38712869e8d6a Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 1 Mar 2021 22:08:33 +0000 Subject: [PATCH 1/2] Fix slow queries --- database.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/database.sql b/database.sql index 12d032254..f298a8e8e 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2021.03-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1406 +-- DB_UPDATE_VERSION 1407 -- ------------------------------------------ @@ -929,7 +929,7 @@ CREATE TABLE IF NOT EXISTS `post` ( `network` char(4) NOT NULL DEFAULT '' COMMENT 'Network from where the item comes from', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the owner of this item', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the author of this item', - `causer-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', + `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', `post-type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Post type (personal note, bookmark, ...)', `vid` smallint unsigned COMMENT 'Id of the verb table entry that contains the activity verbs', `private` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0=public, 1=private, 2=unlisted', @@ -1061,7 +1061,7 @@ CREATE TABLE IF NOT EXISTS `post-thread` ( `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item author', - `causer-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', + `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', `network` char(4) NOT NULL DEFAULT '' COMMENT '', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', @@ -1096,7 +1096,7 @@ CREATE TABLE IF NOT EXISTS `post-user` ( `network` char(4) NOT NULL DEFAULT '' COMMENT 'Network from where the item comes from', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the owner of this item', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the author of this item', - `causer-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', + `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', `post-type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Post type (personal note, bookmark, ...)', `vid` smallint unsigned COMMENT 'Id of the verb table entry that contains the activity verbs', `private` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0=public, 1=private, 2=unlisted', @@ -1155,7 +1155,7 @@ CREATE TABLE IF NOT EXISTS `post-thread-user` ( `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item author', - `causer-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', + `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', `network` char(4) NOT NULL DEFAULT '' COMMENT '', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', From 5f48d6497ee1a2053558a58d1a2b9939e6a61494 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 1 Mar 2021 22:19:47 +0000 Subject: [PATCH 2/2] Fix slow queries --- src/Model/Contact.php | 2 +- src/Model/Item.php | 4 ++++ src/Model/Post.php | 2 +- src/Worker/UpdateContacts.php | 30 ++++++++++++++++++++---------- static/dbstructure.config.php | 10 +++++----- update.php | 16 ++++++++++++++++ 6 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/Model/Contact.php b/src/Model/Contact.php index ae715840e..3b65f2af1 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -1959,7 +1959,7 @@ class Contact } // If Probe::uri fails the network code will be different ("feed" or "unkn") - if (in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]) && ($ret['network'] != $contact['network'])) { + if (($ret['network'] == Protocol::PHANTOM) || (($ret['network'] == Protocol::FEED) && ($ret['network'] != $contact['network']))) { self::updateContact($id, $uid, $contact['url'], $ret['url'], ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]); return false; } diff --git a/src/Model/Item.php b/src/Model/Item.php index 48652a0e1..299736d14 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -970,6 +970,10 @@ class Item unset($item['event-id']); } + if (empty($item['causer-id'])) { + unset($item['causer-id']); + } + Post::insert($item['uri-id'], $item); if ($item['gravity'] == GRAVITY_PARENT) { diff --git a/src/Model/Post.php b/src/Model/Post.php index 002ddbc07..51f74ced5 100644 --- a/src/Model/Post.php +++ b/src/Model/Post.php @@ -285,7 +285,7 @@ class Post $condition = DBA::mergeConditions($condition, ["`visible` AND NOT `deleted` AND NOT `author-blocked` AND NOT `owner-blocked` - AND (NOT `causer-blocked` OR `causer-id` = ?) AND NOT `contact-blocked` + AND (NOT `causer-blocked` OR `causer-id` = ? OR `causer-id` IS NULL) AND NOT `contact-blocked` AND ((NOT `contact-readonly` AND NOT `contact-pending` AND (`contact-rel` IN (?, ?))) OR `self` OR `gravity` != ? OR `contact-uid` = ?) AND NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `hidden` AND `uri-id` = `" . $view . "`.`uri-id` AND `uid` = ?) diff --git a/src/Worker/UpdateContacts.php b/src/Worker/UpdateContacts.php index 699e0f5ab..86959c88b 100644 --- a/src/Worker/UpdateContacts.php +++ b/src/Worker/UpdateContacts.php @@ -49,16 +49,27 @@ class UpdateContacts return; } - // Add every contact our system interacted with and hadn't been updated for a week if unarchived - // or for a month if archived. - $condition = DBA::mergeConditions($base_condition, ["(`id` IN (SELECT `author-id` FROM `post-user`) OR - `id` IN (SELECT `owner-id` FROM `post-user`) OR `id` IN (SELECT `causer-id` FROM `post-user`) OR - `id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`) OR `uid` != ?) AND - (`last-update` < ? OR (NOT `archive` AND `last-update` < ?))", + $condition = DBA::mergeConditions($base_condition, + ["`uid` != ? AND (`last-update` < ? OR (NOT `archive` AND `last-update` < ?))", 0, DateTimeFormat::utc('now - 1 month'), DateTimeFormat::utc('now - 1 week')]); - Logger::info('Updatable interacting federated contacts', ['count' => DBA::count('contact', $condition)]); $ids = self::getContactsToUpdate($condition, [], $limit); - Logger::info('Fetched interacting federated contacts', ['count' => count($ids)]); + Logger::info('Fetched federated user contacts', ['count' => count($ids)]); + + $conditions = ["`id` IN (SELECT `author-id` FROM `post-user`)", "`id` IN (SELECT `owner-id` FROM `post-user`)", + "`id` IN (SELECT `causer-id` FROM `post-user`)", "`id` IN (SELECT `cid` FROM `post-tag`)", + "`id` IN (SELECT `cid` FROM `user-contact`)"]; + + foreach ($conditions as $contact_condition) { + $condition = DBA::mergeConditions($base_condition, + [$contact_condition . " AND (`last-update` < ? OR (NOT `archive` AND `last-update` < ?))", + DateTimeFormat::utc('now - 1 month'), DateTimeFormat::utc('now - 1 week')]); + $ids = self::getContactsToUpdate($condition, $ids, $limit); + Logger::info('Fetched interacting federated contacts', ['count' => count($ids), 'condition' => $contact_condition]); + } + + if (count($ids) > $limit) { + $ids = array_slice($ids, 0, $limit, true); + } if (!DI::config()->get('system', 'update_active_contacts')) { // Add every contact (mostly failed ones) that hadn't been updated for six months @@ -66,7 +77,6 @@ class UpdateContacts $condition = DBA::mergeConditions($base_condition, ["(`last-update` < ? OR (NOT `archive` AND `last-update` < ?))", DateTimeFormat::utc('now - 6 month'), DateTimeFormat::utc('now - 1 month')]); - Logger::info('Updatable federated contacts', ['count' => DBA::count('contact', $condition)]); $previous = count($ids); $ids = self::getContactsToUpdate($condition, $ids, $limit - $previous); Logger::info('Fetched federated contacts', ['count' => count($ids) - $previous]); @@ -93,7 +103,7 @@ class UpdateContacts { $contacts = DBA::select('contact', ['id'], $condition, ['limit' => $limit, 'order' => ['last-update']]); while ($contact = DBA::fetch($contacts)) { - $ids[] = $contact['id']; + $ids[$contact['id']] = $contact['id']; } DBA::close($contacts); return $ids; diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index cb84e0ba9..85be690e2 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1406); + define('DB_UPDATE_VERSION', 1407); } return [ @@ -993,7 +993,7 @@ return [ "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the owner of this item"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the author of this item"], - "causer-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], + "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], "post-type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Post type (personal note, bookmark, ...)"], "vid" => ["type" => "smallint unsigned", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"], "private" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "0=public, 1=private, 2=unlisted"], @@ -1114,7 +1114,7 @@ return [ "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item owner"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item author"], - "causer-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], + "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], @@ -1146,7 +1146,7 @@ return [ "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the owner of this item"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the author of this item"], - "causer-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], + "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], "post-type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Post type (personal note, bookmark, ...)"], "vid" => ["type" => "smallint unsigned", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"], "private" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "0=public, 1=private, 2=unlisted"], @@ -1194,7 +1194,7 @@ return [ "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item owner"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item author"], - "causer-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], + "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], diff --git a/update.php b/update.php index b8d357da2..a1343255b 100644 --- a/update.php +++ b/update.php @@ -885,3 +885,19 @@ function update_1404() DBA::update('workerqueue', ['parameter' => json_encode($parameters)], ['id' => $task['id']]); } } + +function update_1407() +{ + if (!DBA::e("UPDATE `post` SET `causer-id` = NULL WHERE `causer-id` = 0")) { + return Update::FAILED; + } + if (!DBA::e("UPDATE `post-user` SET `causer-id` = NULL WHERE `causer-id` = 0")) { + return Update::FAILED; + } + if (!DBA::e("UPDATE `post-thread` SET `causer-id` = NULL WHERE `causer-id` = 0")) { + return Update::FAILED; + } + if (!DBA::e("UPDATE `post-thread-user` SET `causer-id` = NULL WHERE `causer-id` = 0")) { + return Update::FAILED; + } +}