bitPickups/friendica_2020-09-1_sharedHosting
1
0
Fork 0
Code Pull requests Activity

More "item" traces removed

Browse source
This commit is contained in:
Michael 2021-02-14 09:43:27 +00:00
commit 07c07ec499
15 changed files with 325 additions and 463 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/Model/Item.php
View file

@ -77,7 +77,7 @@ class Item
'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink',
'wall', 'private', 'starred', 'origin', 'title', 'body', 'language',
'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object',
'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'item_id',
'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network',
'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network',
'causer-id', 'causer-link', 'causer-name', 'causer-avatar', 'causer-contact-type',
@ -959,6 +959,10 @@ class Item
$item['deny_gid']
);
if (!empty($item['extid'])) {
$item['external-id'] = ItemURI::getIdByURI($item['extid']);
}
if ($item['verb'] == Activity::ANNOUNCE) {
self::setOwnerforResharedItem($item);
}

8
src/Model/Post.php
View file

@ -94,6 +94,10 @@ class Post
}
}
if (array_key_exists('extid', $row) && is_null($row['extid'])) {
$row['extid'] = '';
}
return $row;
}
@ -526,8 +530,8 @@ class Post
}
}
if (!empty($update_fields)) {
$rows = DBA::selectToArray('post-view', ['id'], $condition, []);
$ids = array_column($rows, 'id');
$rows = DBA::selectToArray('post-view', ['item-id'], $condition, []);
$ids = array_column($rows, 'item-id');
if (!DBA::update('item', $update_fields, ['id' => $ids])) {
DBA::rollback();
Logger::notice('Updating item failed', ['fields' => $update_fields, 'condition' => $condition]);

7
src/Worker/CleanItemUri.php
View file

@ -42,9 +42,10 @@ class CleanItemUri
}
Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` < ?
AND NOT `id` IN (SELECT `uri-id` FROM `item`)
AND NOT `id` IN (SELECT `parent-uri-id` FROM `item`)
AND NOT `id` IN (SELECT `thr-parent-id` FROM `item`)", $item['uri-id']);
AND NOT EXISTS(SELECT `uri-id` FROM `post-user` WHERE `uri-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `parent-uri-id` FROM `post-user` WHERE `parent-uri-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`)", $item['uri-id']);
Logger::notice('Orphaned URI-ID entries removed', ['result' => $ret, 'rows' => DBA::affectedRows()]);
}
}

3
src/Worker/Cron.php
View file

@ -111,9 +111,6 @@ class Cron
// update nodeinfo data
Worker::add(PRIORITY_LOW, 'NodeInfo');
// Repair entries in the database
Worker::add(PRIORITY_LOW, 'RepairDatabase');
Worker::add(PRIORITY_LOW, 'Expire');
Worker::add(PRIORITY_LOW, 'ExpirePosts');

9
src/Worker/Expire.php
View file

@ -55,21 +55,16 @@ class Expire
Logger::info('Deleting orphaned post-content - start');
/// @todo Replace "item with "post-user" in the future when "item" is removed
$condition = ["NOT EXISTS (SELECT `uri-id` FROM `item` WHERE `item`.`uri-id` = `post-content`.`uri-id`)"];
$condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post-content`.`uri-id`)"];
DBA::delete('post-content', $condition);
Logger::info('Orphaned post-content deleted', ['rows' => DBA::affectedRows()]);
Logger::info('Deleting orphaned post-thread - start');
/// @todo Replace "item with "post-user" in the future when "item" is removed
$condition = ["NOT EXISTS (SELECT `uri-id` FROM `item` WHERE `item`.`uri-id` = `post-thread`.`uri-id`)"];
$condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post-thread`.`uri-id`)"];
DBA::delete('post-thread', $condition);
Logger::info('Orphaned item content deleted', ['rows' => DBA::affectedRows()]);
// make this optional as it could have a performance impact on large sites
if (intval(DI::config()->get('system', 'optimize_items'))) {
DBA::e("OPTIMIZE TABLE `item`");
}
Logger::log('Delete expired items - done', Logger::DEBUG);
return;
} elseif (intval($param) > 0) {

29
src/Worker/ExpirePosts.php
View file

@ -50,15 +50,18 @@ class ExpirePosts
Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `post-thread` WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-thread-user` WHERE (`mention` OR `starred` OR `wall` OR `pinned`) AND `uri-id` = `post-thread`.`uri-id`)
AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `origin` AND `uri-id` = `post-thread`.`uri-id`)
AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-category` WHERE `uri-id` = `post-thread`.`uri-id`)
AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-media` WHERE `uri-id` = `post-thread`.`uri-id`)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` INNER JOIN `contact` ON `contact`.`id` = `contact-id` AND `notify_new_posts`
AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-thread-user`
WHERE (`mention` OR `starred` OR `wall` OR `pinned`) AND `uri-id` = `post-thread`.`uri-id`)
AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-category`
WHERE `uri-id` = `post-thread`.`uri-id`)
AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-media`
WHERE `uri-id` = `post-thread`.`uri-id`)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` INNER JOIN `contact` ON `contact`.`id` = `contact-id` AND `notify_new_posts`
WHERE `parent-uri-id` = `post-thread`.`uri-id`)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item`
WHERE (`origin` OR `starred` OR `resource-id` != 0 OR `event-id` != 0 OR `wall` OR `attach` != '' OR `post-type` = ?)
AND `parent-uri-id` = `post-thread`.`uri-id`))
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user`
WHERE (`origin` OR `event-id` != 0 OR `post-type` = ?) AND `parent-uri-id` = `post-thread`.`uri-id`))
AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-content`
WHERE (`resource-id` != 0) AND `uri-id` = `post-thread`.`uri-id`))
ORDER BY `id` LIMIT ?", $expire_days, Item::PT_PERSONAL_NOTE, $limit);
$rows = DBA::affectedRows();
@ -75,11 +78,11 @@ class ExpirePosts
do {
Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` AS `i` WHERE `i`.`uid` != ?
AND `i`.`parent-uri-id` = `item`.`uri-id`)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` AS `i` WHERE `i`.`uid` = ?
AND `i`.`parent-uri-id` = `item`.`uri-id` AND `i`.`received` > UTC_TIMESTAMP() - INTERVAL ? DAY))
(SELECT `uri-id` FROM `post-user` WHERE `gravity` = ? AND `uid` = ? AND `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` != ?
AND `i`.`parent-uri-id` = `post-user`.`uri-id`)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` = ?
AND `i`.`parent-uri-id` = `post-user`.`uri-id` AND `i`.`received` > UTC_TIMESTAMP() - INTERVAL ? DAY))
ORDER BY `id` LIMIT ?",
GRAVITY_PARENT, 0, $expire_days_unclaimed, 0, 0, $expire_days_unclaimed, $limit);

6
src/Worker/RemoveUnusedAvatars.php
View file

@ -35,9 +35,9 @@ class RemoveUnusedAvatars
public static function execute()
{
$condition = ["`uid` = ? AND NOT `self` AND NOT `nurl` IN (SELECT `nurl` FROM `contact` WHERE `uid` != ?)
AND `id` IN (SELECT `contact-id` FROM `photo`) AND NOT `id` IN (SELECT `author-id` FROM `item`)
AND NOT `id` IN (SELECT `owner-id` FROM `item`) AND NOT `id` IN (SELECT `causer-id` FROM `item`)
AND NOT `id` IN (SELECT `cid` FROM `post-tag`) AND NOT `id` IN (SELECT `contact-id` FROM `item`)", 0, 0];
AND `id` IN (SELECT `contact-id` FROM `photo`) AND NOT `id` IN (SELECT `author-id` FROM `post-user`)
AND NOT `id` IN (SELECT `owner-id` FROM `post-user`) AND NOT `id` IN (SELECT `causer-id` FROM `post-user`)
AND NOT `id` IN (SELECT `cid` FROM `post-tag`) AND NOT `id` IN (SELECT `contact-id` FROM `post-user`)", 0, 0];
$total = DBA::count('contact', $condition);
Logger::notice('Starting removal', ['total' => $total]);

28
src/Worker/RemoveUnusedContacts.php
View file

@ -36,11 +36,9 @@ class RemoveUnusedContacts
{
$condition = ["`uid` = ? AND NOT `self` AND NOT `nurl` IN (SELECT `nurl` FROM `contact` WHERE `uid` != ?)
AND (NOT `network` IN (?, ?, ?, ?, ?, ?) OR (`archive` AND `success_update` < UTC_TIMESTAMP() - INTERVAL ? DAY))
AND NOT `id` IN (SELECT `author-id` FROM `item`) AND NOT `id` IN (SELECT `owner-id` FROM `item`)
AND NOT `id` IN (SELECT `causer-id` FROM `item`) AND NOT `id` IN (SELECT `cid` FROM `post-tag`)
AND NOT `id` IN (SELECT `contact-id` FROM `item`) AND NOT `id` IN (SELECT `author-id` FROM `post-thread`)
AND NOT `id` IN (SELECT `owner-id` FROM `post-thread`) AND NOT `id` IN (SELECT `causer-id` FROM `post-thread`)
AND NOT `id` IN (SELECT `contact-id` FROM `post-user`) AND NOT `id` IN (SELECT `cid` FROM `user-contact`)
AND NOT `id` IN (SELECT `author-id` FROM `post-user`) AND NOT `id` IN (SELECT `owner-id` FROM `post-user`)
AND NOT `id` IN (SELECT `causer-id` FROM `post-user`) AND NOT `id` IN (SELECT `cid` FROM `post-tag`)
AND NOT `id` IN (SELECT `contact-id` FROM `post-user`) AND NOT `id` IN (SELECT `cid` FROM `user-contact`)
AND NOT `id` IN (SELECT `cid` FROM `event`) AND NOT `id` IN (SELECT `contact-id` FROM `group_member`)",
0, 0, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED, Protocol::MAIL, Protocol::ACTIVITYPUB, 365];
@ -54,6 +52,26 @@ class RemoveUnusedContacts
DBA::delete('thread', ['owner-id' => $contact['id']]);
DBA::delete('thread', ['author-id' => $contact['id']]);
}
if (DBStructure::existsTable('item')) {
DBA::delete('item', ['owner-id' => $contact['id']]);
DBA::delete('item', ['author-id' => $contact['id']]);
DBA::delete('item', ['causer-id' => $contact['id']]);
}
// There should be none entry for the contact in these tables when none was found in "post-user".
// But we want to be sure since the foreign key prohibits deletion otherwise.
DBA::delete('post', ['owner-id' => $contact['id']]);
DBA::delete('post', ['author-id' => $contact['id']]);
DBA::delete('post', ['causer-id' => $contact['id']]);
DBA::delete('post-thread', ['owner-id' => $contact['id']]);
DBA::delete('post-thread', ['author-id' => $contact['id']]);
DBA::delete('post-thread', ['causer-id' => $contact['id']]);
DBA::delete('post-thread-user', ['owner-id' => $contact['id']]);
DBA::delete('post-thread-user', ['author-id' => $contact['id']]);
DBA::delete('post-thread-user', ['causer-id' => $contact['id']]);
DBA::delete('contact', ['id' => $contact['id']]);
if ((++$count % 1000) == 0) {
Logger::notice('In removal', ['count' => $count, 'total' => $total]);

68
src/Worker/RepairDatabase.php
View file

@ -1,68 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Worker;
use Friendica\Database\DBA;
use Friendica\Model\ItemURI;
/**
* Do some repairs in database entries
* @todo This class can be deleted without replacement when the item table is removed
*/
class RepairDatabase
{
public static function execute()
{
// Ensure that there are no "uri-id", "parent-uri-id" or "thr-parent-id" fields that are NULL
$items = DBA::select('item', ['id', 'uri', 'guid'], ["`uri-id` IS NULL"]);
while ($item = DBA::fetch($items)) {
$uriid = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]);
DBA::update('item', ['uri-id' => $uriid], ['id' => $item['id']]);
}
DBA::close($items);
$items = DBA::select('item', ['id', 'parent-uri'], ["`parent-uri-id` IS NULL"]);
while ($item = DBA::fetch($items)) {
$uriid = ItemURI::getIdByURI($item['parent-uri']);
DBA::update('item', ['parent-uri-id' => $uriid], ['id' => $item['id']]);
}
DBA::close($items);
$items = DBA::select('item', ['id', 'thr-parent'], ["`thr-parent-id` IS NULL"]);
while ($item = DBA::fetch($items)) {
$uriid = ItemURI::getIdByURI($item['thr-parent']);
DBA::update('item', ['thr-parent-id' => $uriid], ['id' => $item['id']]);
}
DBA::close($items);
// Ensure that all uri-id are set correctly
DBA::e("UPDATE `item` INNER JOIN `item-uri` ON `item-uri`.`uri` = `item`.`uri`
SET `uri-id` = `item-uri`.`id` WHERE `item`.`uri-id` != `item-uri`.`id` AND `item`.`uri` != ?", '');
DBA::e("UPDATE `item` INNER JOIN `item-uri` ON `item-uri`.`uri` = `item`.`parent-uri`
SET `parent-uri-id` = `item-uri`.`id` WHERE `item`.`parent-uri-id` != `item-uri`.`id` AND `item`.`parent-uri` != ?", '');
DBA::e("UPDATE `item` INNER JOIN `item-uri` ON `item-uri`.`uri` = `item`.`thr-parent`
SET `thr-parent-id` = `item-uri`.`id` WHERE `item`.`thr-parent-id` != `item-uri`.`id` AND `item`.`thr-parent` != ?", '');
// Delete orphaned data from notify table.
DBA::e("DELETE FROM `notify` WHERE NOT `type` IN (1, 2, 16, 32, 512) AND NOT `iid` IN (SELECT `id` FROM `item`)");
}
}

4
src/Worker/UpdateContacts.php
View file

@ -51,8 +51,8 @@ class UpdateContacts
// Add every contact our system interacted with and hadn't been updated for a week if unarchived
// or for a month if archived.
$condition = DBA::mergeConditions($base_condition, ["(`id` IN (SELECT `author-id` FROM `item`) OR
`id` IN (SELECT `owner-id` FROM `item`) OR `id` IN (SELECT `causer-id` FROM `item`) OR
$condition = DBA::mergeConditions($base_condition, ["(`id` IN (SELECT `author-id` FROM `post-user`) OR
`id` IN (SELECT `owner-id` FROM `post-user`) OR `id` IN (SELECT `causer-id` FROM `post-user`) OR
`id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`) OR `uid` != ?) AND
(`last-update` < ? OR (NOT `archive` AND `last-update` < ?))",
0, DateTimeFormat::utc('now - 1 month'), DateTimeFormat::utc('now - 1 week')]);