From 5a685427aca934934a71b0855c4685169c4cd520 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 17 Oct 2020 08:16:17 +0000 Subject: [PATCH 1/3] Post expiry: Limit for deletions --- src/Worker/ExpirePosts.php | 57 +++++++++++++++++++++++--------------- static/defaults.config.php | 3 +- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index f8d2432679..09684fccc4 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -39,35 +39,48 @@ class ExpirePosts $expire_days_unclaimed = $expire_days; } - if (!empty($expire_days)) { - Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]); - $ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN - (SELECT `uri-id` FROM `thread` - INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts` - WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY - AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin` - AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item` - WHERE (`item`.`starred` OR (`item`.`resource-id` != '') - OR (`item`.`event-id` != '') OR (`item`.`attach` != '') - OR `item`.`wall` OR `item`.`origin` - OR `uri-id` IN (SELECT `uri-id` FROM `post-category` - WHERE `uri-id` = `item`.`uri-id`)) - AND `item`.`parent` = `thread`.`iid`))", $expire_days); + $limit = DI::config()->get('system', 'dbclean-expire-limit'); + if (empty($limit)) { + return; + } - Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => DBA::affectedRows()]); + if (!empty($expire_days)) { + do { + Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]); + $ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN + (SELECT `uri-id` FROM `thread` + INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts` + WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY + AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin` + AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item` + WHERE (`item`.`starred` OR (`item`.`resource-id` != '') + OR (`item`.`event-id` != '') OR (`item`.`attach` != '') + OR `item`.`wall` OR `item`.`origin` + OR `uri-id` IN (SELECT `uri-id` FROM `post-category` + WHERE `uri-id` = `item`.`uri-id`)) + AND `item`.`parent` = `thread`.`iid`)) + ORDER BY `id` LIMIT ?", $expire_days, $limit); + + $rows = DBA::affectedRows(); + Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => $rows]); + } while ($rows >= $limit); } if (!empty($expire_days_unclaimed)) { $expiry_date = DateTimeFormat::utc('now - ' . $expire_days_unclaimed . ' days', DateTimeFormat::MYSQL); - Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]); - $ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN - (SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ? - AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?) - AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?))", - GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date); + do { + Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]); + $ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN + (SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ? + AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?) + AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?)) + ORDER BY `id` LIMIT ?", + GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date, $limit); - Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => DBA::affectedRows()]); + $rows = DBA::affectedRows(); + Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => $rows]); + } while ($rows >= $limit); } } } diff --git a/static/defaults.config.php b/static/defaults.config.php index 18b70d3c15..4174ebcad9 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -187,7 +187,8 @@ return [ // dbclean-expire-limit (Integer) // This defines the number of items that are to be deleted in a single call. - // Reduce this value when you are getting memory issues. + // Reduce this value when you are getting lock issues. + // A value of 0 disables the deletion process. 'dbclean-expire-limit' => 1000, // diaspora_test (Boolean) From 7efd01880c07934174df98342c511b05c28a4dd7 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 17 Oct 2020 12:39:42 +0000 Subject: [PATCH 2/3] Define a maintenance window --- src/Core/Worker.php | 44 ++++++++++++++++++++++++++++++++++++++ src/Worker/Cron.php | 4 ++-- src/Worker/ExpirePosts.php | 11 ++++++++++ static/defaults.config.php | 10 +++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 426389ab00..9ab061baac 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -1394,4 +1394,48 @@ class Worker return (bool)$row['jobs']; } + + /** + * Check if the system is inside the defined maintenance window + * + * @return boolean + */ + public static function isInMaintenanceWindow(bool $check_last_execution = false) + { + // Calculate the seconds of the start end end of the maintenance window + $start = strtotime(DI::config()->get('system', 'maintenance_start')) % 86400; + $end = strtotime(DI::config()->get('system', 'maintenance_end')) % 86400; + + Logger::info('Maintenance window', ['start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]); + + if ($check_last_execution) { + // Calculate the window duration + $duration = max($start, $end) - min($start, $end); + + // Quit when the last cron execution had been after the previous window + $last_cron = DI::config()->get('system', 'last_cron_daily'); + if ($last_cron + $duration > time()) { + Logger::info('The Daily cron had been executed recently', ['last' => date(DateTimeFormat::MYSQL, $last_cron), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]); + return false; + } + } + + $current = time() % 86400; + + if ($start < $end) { + // Execute if we are inside the window + $execute = ($current >= $start) && ($current <= $end); + } else { + // Don't execute if we are outside the window + $execute = !(($current > $end) && ($current < $start)); + } + + if (!$execute) { + Logger::info('We are outside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]); + } else { + Logger::info('We are inside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]); + } + + return $execute; + } } diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 90042e30f6..3ce5fb4605 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -93,8 +93,8 @@ class Cron DI::config()->set('system', 'last_cron_hourly', time()); } - // Daily cron calls - if (DI::config()->get('system', 'last_cron_daily', 0) + 86400 < time()) { + // Daily maintenance cron calls + if (Worker::isInMaintenanceWindow(true)) { Worker::add(PRIORITY_LOW, 'UpdateContactBirthdays'); diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index 09684fccc4..3a49453257 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -22,6 +22,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; +use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Util\DateTimeFormat; @@ -63,6 +64,11 @@ class ExpirePosts $rows = DBA::affectedRows(); Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => $rows]); + + if (!Worker::isInMaintenanceWindow()) { + Logger::notice('We are outside of the maintenance window, quitting'); + return; + } } while ($rows >= $limit); } @@ -80,6 +86,11 @@ class ExpirePosts $rows = DBA::affectedRows(); Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => $rows]); + + if (!Worker::isInMaintenanceWindow()) { + Logger::notice('We are outside of the maintenance window, quitting'); + return; + } } while ($rows >= $limit); } } diff --git a/static/defaults.config.php b/static/defaults.config.php index 4174ebcad9..b0832d88f3 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -300,6 +300,16 @@ return [ // Sets the logging adapter of Friendica globally (monolog, syslog, stream) 'logger_config' => 'stream', + // maintenance_start (String) + // Start of the window for the daily maintenance cron call. + // The system timezone is used when no timezone is defined here. + 'maintenance_start' => '01:00 +00:00', + + // maintenance_end (String) + // End of the window for the daily maintenance cron call + // The system timezone is used when no timezone is defined here. + 'maintenance_end' => '03:00 +00:00', + // max_batch_queue (Integer) // Maximum number of batched queue items for a single contact before subsequent messages are discarded. 'max_batch_queue' => 1000, From 2c0db7546ba59465c1dc6af1be00efe7677ced4b Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 17 Oct 2020 15:54:52 +0200 Subject: [PATCH 3/3] Update src/Core/Worker.php Co-authored-by: Hypolite Petovan --- src/Core/Worker.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 9ab061baac..09cf14e5af 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -1430,10 +1430,10 @@ class Worker $execute = !(($current > $end) && ($current < $start)); } - if (!$execute) { - Logger::info('We are outside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]); - } else { + if ($execute) { Logger::info('We are inside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]); + } else { + Logger::info('We are outside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]); } return $execute;