From a228183abbf8aba00318d358d5b9a69932ea6247 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 2 Mar 2021 06:50:12 +0000 Subject: [PATCH 1/2] Call post and item-uri expiry in squence --- src/Worker/CleanItemUri.php | 60 ---------------------- src/Worker/Cron.php | 7 --- src/Worker/Expire.php | 65 +++++------------------ src/Worker/ExpirePosts.php | 100 +++++++++++++++++++++++++++++++++++- 4 files changed, 112 insertions(+), 120 deletions(-) delete mode 100644 src/Worker/CleanItemUri.php diff --git a/src/Worker/CleanItemUri.php b/src/Worker/CleanItemUri.php deleted file mode 100644 index 84ef537c4..000000000 --- a/src/Worker/CleanItemUri.php +++ /dev/null @@ -1,60 +0,0 @@ -. - * - */ - -namespace Friendica\Worker; - -use Friendica\Core\Logger; -use Friendica\Database\DBA; -use Friendica\Model\Post; - -class CleanItemUri -{ - /** - * Delete unused item-uri entries - */ - public static function execute() - { - // We have to avoid deleting newly created "item-uri" entries. - // So we fetch a post that had been stored yesterday and only delete older ones. - $item = Post::selectFirst(['uri-id'], ["`uid` = ? AND `received` < UTC_TIMESTAMP() - INTERVAL ? DAY", 0, 1], - ['order' => ['received' => true]]); - if (empty($item['uri-id'])) { - Logger::warning('No item with uri-id found - we better quit here'); - return; - } - Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]); - $uris = DBA::select('item-uri', ['id'], ["`id` < ? - AND NOT EXISTS(SELECT `uri-id` FROM `post` WHERE `uri-id` = `item-uri`.`id`) - AND NOT EXISTS(SELECT `parent-uri-id` FROM `post` WHERE `parent-uri-id` = `item-uri`.`id`) - AND NOT EXISTS(SELECT `thr-parent-id` FROM `post` WHERE `thr-parent-id` = `item-uri`.`id`) - AND NOT EXISTS(SELECT `external-id` FROM `post` WHERE `external-id` = `item-uri`.`id`)", $item['uri-id']]); - - $affected_count = 0; - while ($rows = DBA::toArray($uris, false, 100)) { - $ids = array_column($rows, 'id'); - DBA::delete('item-uri', ['id' => $ids]); - $affected_count += DBA::affectedRows(); - Logger::info('Deleted', ['rows' => $affected_count]); - } - DBA::close($uris); - Logger::notice('Orphaned URI-ID entries removed', ['rows' => $affected_count]); - } -} diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 93e0215f7..c8ee97d0c 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -108,17 +108,10 @@ class Cron Worker::add(PRIORITY_LOW, 'UpdatePhotoAlbums'); - // update nodeinfo data - Worker::add(PRIORITY_LOW, 'NodeInfo'); - Worker::add(PRIORITY_LOW, 'Expire'); - Worker::add(PRIORITY_LOW, 'ExpirePosts'); - Worker::add(PRIORITY_LOW, 'ExpireConversations'); - Worker::add(PRIORITY_LOW, 'CleanItemUri'); - Worker::add(PRIORITY_LOW, 'RemoveUnusedContacts'); Worker::add(PRIORITY_LOW, 'RemoveUnusedAvatars'); diff --git a/src/Worker/Expire.php b/src/Worker/Expire.php index 62c7097f4..769b0d048 100644 --- a/src/Worker/Expire.php +++ b/src/Worker/Expire.php @@ -25,10 +25,8 @@ use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Database\DBA; -use Friendica\Database\DBStructure; use Friendica\DI; use Friendica\Model\Item; -use Friendica\Model\Post; /** * Expires old item entries @@ -41,81 +39,44 @@ class Expire Hook::loadHooks(); - if ($param == 'delete') { - Logger::log('Delete expired items', Logger::DEBUG); - // physically remove anything that has been deleted for more than two months - $condition = ["`gravity` = ? AND `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY", GRAVITY_PARENT]; - $rows = Post::select(['guid', 'uri-id', 'uid'], $condition); - while ($row = Post::fetch($rows)) { - Logger::info('Delete expired item', ['uri-id' => $row['uri-id'], 'guid' => $row['guid']]); - if (DBStructure::existsTable('item')) { - DBA::delete('item', ['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]); - } - Post\User::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]); - } - DBA::close($rows); - - Logger::info('Deleting orphaned post entries- start'); - $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post`.`uri-id`)"]; - DBA::delete('post', $condition); - Logger::info('Orphaned post entries deleted', ['rows' => DBA::affectedRows()]); - - Logger::info('Deleting orphaned post-content entries - start'); - $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post-content`.`uri-id`)"]; - DBA::delete('post-content', $condition); - Logger::info('Orphaned post-content entries deleted', ['rows' => DBA::affectedRows()]); - - Logger::info('Deleting orphaned post-thread entries - start'); - $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post-thread`.`uri-id`)"]; - DBA::delete('post-thread', $condition); - Logger::info('Orphaned post-thread entries deleted', ['rows' => DBA::affectedRows()]); - - Logger::info('Deleting orphaned post-thread-user entries - start'); - $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post-thread-user`.`uri-id`)"]; - DBA::delete('post-thread-user', $condition); - Logger::info('Orphaned post-thread-user entries deleted', ['rows' => DBA::affectedRows()]); - - Logger::log('Delete expired items - done', Logger::DEBUG); - return; - } elseif (intval($param) > 0) { + if (intval($param) > 0) { $user = DBA::selectFirst('user', ['uid', 'username', 'expire'], ['uid' => $param]); if (DBA::isResult($user)) { - Logger::log('Expire items for user '.$user['uid'].' ('.$user['username'].') - interval: '.$user['expire'], Logger::DEBUG); + Logger::info('Expire items', ['user' => $user['uid'], 'username' => $user['username'], 'interval' => $user['expire']]); Item::expire($user['uid'], $user['expire']); - Logger::log('Expire items for user '.$user['uid'].' ('.$user['username'].') - done ', Logger::DEBUG); + Logger::info('Expire items done', ['user' => $user['uid'], 'username' => $user['username'], 'interval' => $user['expire']]); } return; } elseif ($param == 'hook' && !empty($hook_function)) { foreach (Hook::getByName('expire') as $hook) { if ($hook[1] == $hook_function) { - Logger::log("Calling expire hook '" . $hook[1] . "'", Logger::DEBUG); + Logger::info('Calling expire hook', ['hook' => $hook[1]]); Hook::callSingle($a, 'expire', $hook, $data); } } return; } - Logger::log('expire: start'); - - Worker::add(['priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true], - 'Expire', 'delete'); + Logger::notice('start expiry'); $r = DBA::p("SELECT `uid`, `username` FROM `user` WHERE `expire` != 0"); while ($row = DBA::fetch($r)) { - Logger::log('Calling expiry for user '.$row['uid'].' ('.$row['username'].')', Logger::DEBUG); + Logger::info('Calling expiry', ['user' => $row['uid'], 'username' => $row['username']]); Worker::add(['priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true], - 'Expire', (int)$row['uid']); + 'Expire', (int)$row['uid']); } DBA::close($r); - Logger::log('expire: calling hooks'); + Logger::notice('calling hooks'); foreach (Hook::getByName('expire') as $hook) { - Logger::log("Calling expire hook for '" . $hook[1] . "'", Logger::DEBUG); + Logger::info('Calling expire', ['hook' => $hook[1]]); Worker::add(['priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true], - 'Expire', 'hook', $hook[1]); + 'Expire', 'hook', $hook[1]); } - Logger::log('expire: end'); + Logger::notice('calling hooks done'); + + Worker::add(['priority' => $a->queue['priority'], 'dont_fork' => true], 'ExpirePosts'); return; } diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index 14fb560b1..9e75ea66b 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -22,16 +22,114 @@ namespace Friendica\Worker; use Friendica\Core\Logger; +use Friendica\Core\Worker; use Friendica\Database\DBA; +use Friendica\Database\DBStructure; use Friendica\DI; use Friendica\Model\Item; +use Friendica\Model\Post; class ExpirePosts { /** - * Delete old post entries + * Expire posts and remove unused item-uri entries + * + * @return void */ public static function execute() + { + $a = DI::app(); + + self::deleteExpiredOriginPosts(); + + self::deleteUnusedItemUri(); + + self::deleteExpiredExternalPosts(); + + // update nodeinfo data after everything is cleaned up + Worker::add(['priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true], 'NodeInfo'); + } + + /** + * Delete expired origin posts and orphaned post related table entries + * + * @return void + */ + private static function deleteExpiredOriginPosts() + { + Logger::info('Delete expired posts'); + // physically remove anything that has been deleted for more than two months + $condition = ["`gravity` = ? AND `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY", GRAVITY_PARENT]; + $rows = Post::select(['guid', 'uri-id', 'uid'], $condition); + while ($row = Post::fetch($rows)) { + Logger::info('Delete expired item', ['uri-id' => $row['uri-id'], 'guid' => $row['guid']]); + if (DBStructure::existsTable('item')) { + DBA::delete('item', ['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]); + } + Post\User::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]); + } + DBA::close($rows); + + Logger::info('Deleting orphaned post entries- start'); + $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post`.`uri-id`)"]; + DBA::delete('post', $condition); + Logger::info('Orphaned post entries deleted', ['rows' => DBA::affectedRows()]); + + Logger::info('Deleting orphaned post-content entries - start'); + $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post-content`.`uri-id`)"]; + DBA::delete('post-content', $condition); + Logger::info('Orphaned post-content entries deleted', ['rows' => DBA::affectedRows()]); + + Logger::info('Deleting orphaned post-thread entries - start'); + $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post-thread`.`uri-id`)"]; + DBA::delete('post-thread', $condition); + Logger::info('Orphaned post-thread entries deleted', ['rows' => DBA::affectedRows()]); + + Logger::info('Deleting orphaned post-thread-user entries - start'); + $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post-thread-user`.`uri-id`)"]; + DBA::delete('post-thread-user', $condition); + Logger::info('Orphaned post-thread-user entries deleted', ['rows' => DBA::affectedRows()]); + + Logger::info('Delete expired posts - done'); + } + + /** + * Delete unused item-uri entries + */ + private static function deleteUnusedItemUri() + { + $a = DI::app(); + + // We have to avoid deleting newly created "item-uri" entries. + // So we fetch a post that had been stored yesterday and only delete older ones. + $item = Post::selectFirst(['uri-id'], ["`uid` = ? AND `received` < UTC_TIMESTAMP() - INTERVAL ? DAY", 0, 1], + ['order' => ['received' => true]]); + if (empty($item['uri-id'])) { + Logger::warning('No item with uri-id found - we better quit here'); + return; + } + Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]); + $uris = DBA::select('item-uri', ['id'], ["`id` < ? + AND NOT EXISTS(SELECT `uri-id` FROM `post` WHERE `uri-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `parent-uri-id` FROM `post` WHERE `parent-uri-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `thr-parent-id` FROM `post` WHERE `thr-parent-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `external-id` FROM `post` WHERE `external-id` = `item-uri`.`id`)", $item['uri-id']]); + + $affected_count = 0; + while ($rows = DBA::toArray($uris, false, 100)) { + $ids = array_column($rows, 'id'); + DBA::delete('item-uri', ['id' => $ids]); + $affected_count += DBA::affectedRows(); + Logger::info('Deleted', ['rows' => $affected_count]); + } + DBA::close($uris); + Logger::notice('Orphaned URI-ID entries removed', ['rows' => $affected_count]); + } + + /** + * Delete old external post entries + */ + private static function deleteExpiredExternalPosts() { $expire_days = DI::config()->get('system', 'dbclean-expire-days'); $expire_days_unclaimed = DI::config()->get('system', 'dbclean-expire-unclaimed'); From c740509a222a261146150bfa5cfa2b13ab906156 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 2 Mar 2021 07:06:22 +0000 Subject: [PATCH 2/2] Changed worker order --- src/Worker/Cron.php | 2 +- src/Worker/Expire.php | 2 -- src/Worker/ExpirePosts.php | 9 +++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index c8ee97d0c..a0144e8a2 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -108,7 +108,7 @@ class Cron Worker::add(PRIORITY_LOW, 'UpdatePhotoAlbums'); - Worker::add(PRIORITY_LOW, 'Expire'); + Worker::add(PRIORITY_LOW, 'ExpirePosts'); Worker::add(PRIORITY_LOW, 'ExpireConversations'); diff --git a/src/Worker/Expire.php b/src/Worker/Expire.php index 769b0d048..1fb162db9 100644 --- a/src/Worker/Expire.php +++ b/src/Worker/Expire.php @@ -76,8 +76,6 @@ class Expire Logger::notice('calling hooks done'); - Worker::add(['priority' => $a->queue['priority'], 'dont_fork' => true], 'ExpirePosts'); - return; } } diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index 9e75ea66b..1130a62e4 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -38,16 +38,17 @@ class ExpirePosts */ public static function execute() { - $a = DI::app(); - self::deleteExpiredOriginPosts(); self::deleteUnusedItemUri(); self::deleteExpiredExternalPosts(); + // Set the expiry for origin posta + Worker::add(PRIORITY_LOW, 'Expire'); + // update nodeinfo data after everything is cleaned up - Worker::add(['priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true], 'NodeInfo'); + Worker::add(PRIORITY_LOW, 'NodeInfo'); } /** @@ -70,7 +71,7 @@ class ExpirePosts } DBA::close($rows); - Logger::info('Deleting orphaned post entries- start'); + Logger::info('Deleting orphaned post entries - start'); $condition = ["NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `post-user`.`uri-id` = `post`.`uri-id`)"]; DBA::delete('post', $condition); Logger::info('Orphaned post entries deleted', ['rows' => DBA::affectedRows()]);