bitPickups/friendica_2021.01_tupambae_shared_hosting
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #9974 from annando/slow-queries

Browse source 
Fix slow queries
This commit is contained in:
Hypolite Petovan 2021-03-01 18:11:48 -05:00 committed by GitHub
parent 5b655a9768 5f48d6497e
commit ed3f968487
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 52 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------ -- ------------------------------------------
-- Friendica 2021.03-dev (Red Hot Poker) -- Friendica 2021.03-dev (Red Hot Poker)
-- DB_UPDATE_VERSION 1406 -- DB_UPDATE_VERSION 1407
-- ------------------------------------------ -- ------------------------------------------
@ -929,7 +929,7 @@ CREATE TABLE IF NOT EXISTS `post` (
`network` char(4) NOT NULL DEFAULT '' COMMENT 'Network from where the item comes from', `network` char(4) NOT NULL DEFAULT '' COMMENT 'Network from where the item comes from',
`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the owner of this item', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the owner of this item',
`author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the author of this item', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the author of this item',
`causer-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation',
`post-type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Post type (personal note, bookmark, ...)', `post-type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Post type (personal note, bookmark, ...)',
`vid` smallint unsigned COMMENT 'Id of the verb table entry that contains the activity verbs', `vid` smallint unsigned COMMENT 'Id of the verb table entry that contains the activity verbs',
`private` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0=public, 1=private, 2=unlisted', `private` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0=public, 1=private, 2=unlisted',
@ -1061,7 +1061,7 @@ CREATE TABLE IF NOT EXISTS `post-thread` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner',
`author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item author', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item author',
`causer-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation',
`network` char(4) NOT NULL DEFAULT '' COMMENT '', `network` char(4) NOT NULL DEFAULT '' COMMENT '',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
`received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
@ -1096,7 +1096,7 @@ CREATE TABLE IF NOT EXISTS `post-user` (
`network` char(4) NOT NULL DEFAULT '' COMMENT 'Network from where the item comes from', `network` char(4) NOT NULL DEFAULT '' COMMENT 'Network from where the item comes from',
`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the owner of this item', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the owner of this item',
`author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the author of this item', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the author of this item',
`causer-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation',
`post-type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Post type (personal note, bookmark, ...)', `post-type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Post type (personal note, bookmark, ...)',
`vid` smallint unsigned COMMENT 'Id of the verb table entry that contains the activity verbs', `vid` smallint unsigned COMMENT 'Id of the verb table entry that contains the activity verbs',
`private` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0=public, 1=private, 2=unlisted', `private` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0=public, 1=private, 2=unlisted',
@ -1155,7 +1155,7 @@ CREATE TABLE IF NOT EXISTS `post-thread-user` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner',
`author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item author', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item author',
`causer-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation',
`network` char(4) NOT NULL DEFAULT '' COMMENT '', `network` char(4) NOT NULL DEFAULT '' COMMENT '',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
`received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',

2
src/Model/Contact.php
View file

@ -1959,7 +1959,7 @@ class Contact
} }
// If Probe::uri fails the network code will be different ("feed" or "unkn") // If Probe::uri fails the network code will be different ("feed" or "unkn")
if (in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]) && ($ret['network'] != $contact['network'])) { if (($ret['network'] == Protocol::PHANTOM) || (($ret['network'] == Protocol::FEED) && ($ret['network'] != $contact['network']))) {
self::updateContact($id, $uid, $contact['url'], $ret['url'], ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]); self::updateContact($id, $uid, $contact['url'], $ret['url'], ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]);
return false; return false;
} }

4
src/Model/Item.php
View file

@ -970,6 +970,10 @@ class Item
unset($item['event-id']); unset($item['event-id']);
} }
if (empty($item['causer-id'])) {
unset($item['causer-id']);
}
Post::insert($item['uri-id'], $item); Post::insert($item['uri-id'], $item);
if ($item['gravity'] == GRAVITY_PARENT) { if ($item['gravity'] == GRAVITY_PARENT) {

2
src/Model/Post.php
View file

@ -285,7 +285,7 @@ class Post
$condition = DBA::mergeConditions($condition, $condition = DBA::mergeConditions($condition,
["`visible` AND NOT `deleted` ["`visible` AND NOT `deleted`
AND NOT `author-blocked` AND NOT `owner-blocked` AND NOT `author-blocked` AND NOT `owner-blocked`
AND (NOT `causer-blocked` OR `causer-id` = ?) AND NOT `contact-blocked` AND (NOT `causer-blocked` OR `causer-id` = ? OR `causer-id` IS NULL) AND NOT `contact-blocked`
AND ((NOT `contact-readonly` AND NOT `contact-pending` AND (`contact-rel` IN (?, ?))) AND ((NOT `contact-readonly` AND NOT `contact-pending` AND (`contact-rel` IN (?, ?)))
OR `self` OR `gravity` != ? OR `contact-uid` = ?) OR `self` OR `gravity` != ? OR `contact-uid` = ?)
AND NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `hidden` AND `uri-id` = `" . $view . "`.`uri-id` AND `uid` = ?) AND NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `hidden` AND `uri-id` = `" . $view . "`.`uri-id` AND `uid` = ?)

30
src/Worker/UpdateContacts.php
View file

@ -49,16 +49,27 @@ class UpdateContacts
return; return;
} }
// Add every contact our system interacted with and hadn't been updated for a week if unarchived $condition = DBA::mergeConditions($base_condition,
// or for a month if archived. ["`uid` != ? AND (`last-update` < ? OR (NOT `archive` AND `last-update` < ?))",
$condition = DBA::mergeConditions($base_condition, ["(`id` IN (SELECT `author-id` FROM `post-user`) OR
`id` IN (SELECT `owner-id` FROM `post-user`) OR `id` IN (SELECT `causer-id` FROM `post-user`) OR
`id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`) OR `uid` != ?) AND
(`last-update` < ? OR (NOT `archive` AND `last-update` < ?))",
0, DateTimeFormat::utc('now - 1 month'), DateTimeFormat::utc('now - 1 week')]); 0, DateTimeFormat::utc('now - 1 month'), DateTimeFormat::utc('now - 1 week')]);
Logger::info('Updatable interacting federated contacts', ['count' => DBA::count('contact', $condition)]);
$ids = self::getContactsToUpdate($condition, [], $limit); $ids = self::getContactsToUpdate($condition, [], $limit);
Logger::info('Fetched interacting federated contacts', ['count' => count($ids)]); Logger::info('Fetched federated user contacts', ['count' => count($ids)]);
$conditions = ["`id` IN (SELECT `author-id` FROM `post-user`)", "`id` IN (SELECT `owner-id` FROM `post-user`)",
"`id` IN (SELECT `causer-id` FROM `post-user`)", "`id` IN (SELECT `cid` FROM `post-tag`)",
"`id` IN (SELECT `cid` FROM `user-contact`)"];
foreach ($conditions as $contact_condition) {
$condition = DBA::mergeConditions($base_condition,
[$contact_condition . " AND (`last-update` < ? OR (NOT `archive` AND `last-update` < ?))",
DateTimeFormat::utc('now - 1 month'), DateTimeFormat::utc('now - 1 week')]);
$ids = self::getContactsToUpdate($condition, $ids, $limit);
Logger::info('Fetched interacting federated contacts', ['count' => count($ids), 'condition' => $contact_condition]);
}
if (count($ids) > $limit) {
$ids = array_slice($ids, 0, $limit, true);
}
if (!DI::config()->get('system', 'update_active_contacts')) { if (!DI::config()->get('system', 'update_active_contacts')) {
// Add every contact (mostly failed ones) that hadn't been updated for six months // Add every contact (mostly failed ones) that hadn't been updated for six months
@ -66,7 +77,6 @@ class UpdateContacts
$condition = DBA::mergeConditions($base_condition, $condition = DBA::mergeConditions($base_condition,
["(`last-update` < ? OR (NOT `archive` AND `last-update` < ?))", ["(`last-update` < ? OR (NOT `archive` AND `last-update` < ?))",
DateTimeFormat::utc('now - 6 month'), DateTimeFormat::utc('now - 1 month')]); DateTimeFormat::utc('now - 6 month'), DateTimeFormat::utc('now - 1 month')]);
Logger::info('Updatable federated contacts', ['count' => DBA::count('contact', $condition)]);
$previous = count($ids); $previous = count($ids);
$ids = self::getContactsToUpdate($condition, $ids, $limit - $previous); $ids = self::getContactsToUpdate($condition, $ids, $limit - $previous);
Logger::info('Fetched federated contacts', ['count' => count($ids) - $previous]); Logger::info('Fetched federated contacts', ['count' => count($ids) - $previous]);
@ -93,7 +103,7 @@ class UpdateContacts
{ {
$contacts = DBA::select('contact', ['id'], $condition, ['limit' => $limit, 'order' => ['last-update']]); $contacts = DBA::select('contact', ['id'], $condition, ['limit' => $limit, 'order' => ['last-update']]);
while ($contact = DBA::fetch($contacts)) { while ($contact = DBA::fetch($contacts)) {
$ids[] = $contact['id']; $ids[$contact['id']] = $contact['id'];
} }
DBA::close($contacts); DBA::close($contacts);
return $ids; return $ids;

10
static/dbstructure.config.php
View file

@ -55,7 +55,7 @@
use Friendica\Database\DBA; use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) { if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1406); define('DB_UPDATE_VERSION', 1407);
} }
return [ return [
@ -993,7 +993,7 @@ return [
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"],
"owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the owner of this item"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the owner of this item"],
"author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the author of this item"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the author of this item"],
"causer-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"],
"post-type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Post type (personal note, bookmark, ...)"], "post-type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Post type (personal note, bookmark, ...)"],
"vid" => ["type" => "smallint unsigned", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"], "vid" => ["type" => "smallint unsigned", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"],
"private" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "0=public, 1=private, 2=unlisted"], "private" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "0=public, 1=private, 2=unlisted"],
@ -1114,7 +1114,7 @@ return [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item owner"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item owner"],
"author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item author"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item author"],
"causer-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"],
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
"received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
@ -1146,7 +1146,7 @@ return [
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"],
"owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the owner of this item"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the owner of this item"],
"author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the author of this item"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the author of this item"],
"causer-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"],
"post-type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Post type (personal note, bookmark, ...)"], "post-type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Post type (personal note, bookmark, ...)"],
"vid" => ["type" => "smallint unsigned", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"], "vid" => ["type" => "smallint unsigned", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"],
"private" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "0=public, 1=private, 2=unlisted"], "private" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "0=public, 1=private, 2=unlisted"],
@ -1194,7 +1194,7 @@ return [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item owner"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item owner"],
"author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item author"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item author"],
"causer-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"],
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
"received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],

16
update.php
View file

@ -885,3 +885,19 @@ function update_1404()
DBA::update('workerqueue', ['parameter' => json_encode($parameters)], ['id' => $task['id']]); DBA::update('workerqueue', ['parameter' => json_encode($parameters)], ['id' => $task['id']]);
} }
} }
function update_1407()
{
if (!DBA::e("UPDATE `post` SET `causer-id` = NULL WHERE `causer-id` = 0")) {
return Update::FAILED;
}
if (!DBA::e("UPDATE `post-user` SET `causer-id` = NULL WHERE `causer-id` = 0")) {
return Update::FAILED;
}
if (!DBA::e("UPDATE `post-thread` SET `causer-id` = NULL WHERE `causer-id` = 0")) {
return Update::FAILED;
}
if (!DBA::e("UPDATE `post-thread-user` SET `causer-id` = NULL WHERE `causer-id` = 0")) {
return Update::FAILED;
}
}