strk/friendica
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #9434 from annando/dbclean-limit

Browse source 
Post expiry: Limit for deletions
This commit is contained in:
Hypolite Petovan 2020-10-17 15:14:48 -04:00 committed by GitHub
parent d1244f71fb 009bf0d3ec
commit b3e5621d37
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 104 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
src/Core/Worker.php
View file

@ -1394,4 +1394,48 @@ class Worker
return (bool)$row['jobs'];
}
/**
* Check if the system is inside the defined maintenance window
*
* @return boolean
*/
public static function isInMaintenanceWindow(bool $check_last_execution = false)
{
// Calculate the seconds of the start end end of the maintenance window
$start = strtotime(DI::config()->get('system', 'maintenance_start')) % 86400;
$end = strtotime(DI::config()->get('system', 'maintenance_end')) % 86400;
Logger::info('Maintenance window', ['start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
if ($check_last_execution) {
// Calculate the window duration
$duration = max($start, $end) - min($start, $end);
// Quit when the last cron execution had been after the previous window
$last_cron = DI::config()->get('system', 'last_cron_daily');
if ($last_cron + $duration > time()) {
Logger::info('The Daily cron had been executed recently', ['last' => date(DateTimeFormat::MYSQL, $last_cron), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
return false;
}
}
$current = time() % 86400;
if ($start < $end) {
// Execute if we are inside the window
$execute = ($current >= $start) && ($current <= $end);
} else {
// Don't execute if we are outside the window
$execute = !(($current > $end) && ($current < $start));
}
if ($execute) {
Logger::info('We are inside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
} else {
Logger::info('We are outside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
}
return $execute;
}
}

4
src/Worker/Cron.php
View file

@ -93,8 +93,8 @@ class Cron
DI::config()->set('system', 'last_cron_hourly', time());
}
// Daily cron calls
if (DI::config()->get('system', 'last_cron_daily', 0) + 86400 < time()) {
// Daily maintenance cron calls
if (Worker::isInMaintenanceWindow(true)) {
Worker::add(PRIORITY_LOW, 'UpdateContactBirthdays');

68
src/Worker/ExpirePosts.php
View file

@ -22,6 +22,7 @@
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\DateTimeFormat;
@ -39,35 +40,58 @@ class ExpirePosts
$expire_days_unclaimed = $expire_days;
}
if (!empty($expire_days)) {
Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `thread`
INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts`
WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin`
AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item`
WHERE (`item`.`starred` OR (`item`.`resource-id` != '')
OR (`item`.`event-id` != '') OR (`item`.`attach` != '')
OR `item`.`wall` OR `item`.`origin`
OR `uri-id` IN (SELECT `uri-id` FROM `post-category`
WHERE `uri-id` = `item`.`uri-id`))
AND `item`.`parent` = `thread`.`iid`))", $expire_days);
$limit = DI::config()->get('system', 'dbclean-expire-limit');
if (empty($limit)) {
return;
}
Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => DBA::affectedRows()]);
if (!empty($expire_days)) {
do {
Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `thread`
INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts`
WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin`
AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item`
WHERE (`item`.`starred` OR (`item`.`resource-id` != '')
OR (`item`.`event-id` != '') OR (`item`.`attach` != '')
OR `item`.`wall` OR `item`.`origin`
OR `uri-id` IN (SELECT `uri-id` FROM `post-category`
WHERE `uri-id` = `item`.`uri-id`))
AND `item`.`parent` = `thread`.`iid`))
ORDER BY `id` LIMIT ?", $expire_days, $limit);
$rows = DBA::affectedRows();
Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => $rows]);
if (!Worker::isInMaintenanceWindow()) {
Logger::notice('We are outside of the maintenance window, quitting');
return;
}
} while ($rows >= $limit);
}
if (!empty($expire_days_unclaimed)) {
$expiry_date = DateTimeFormat::utc('now - ' . $expire_days_unclaimed . ' days', DateTimeFormat::MYSQL);
Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?))",
GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date);
do {
Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?))
ORDER BY `id` LIMIT ?",
GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date, $limit);
Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => DBA::affectedRows()]);
$rows = DBA::affectedRows();
Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => $rows]);
if (!Worker::isInMaintenanceWindow()) {
Logger::notice('We are outside of the maintenance window, quitting');
return;
}
} while ($rows >= $limit);
}
}
}

13
static/defaults.config.php
View file

@ -187,7 +187,8 @@ return [
// dbclean-expire-limit (Integer)
// This defines the number of items that are to be deleted in a single call.
// Reduce this value when you are getting memory issues.
// Reduce this value when you are getting lock issues.
// A value of 0 disables the deletion process.
'dbclean-expire-limit' => 1000,
// diaspora_test (Boolean)
@ -299,6 +300,16 @@ return [
// Sets the logging adapter of Friendica globally (monolog, syslog, stream)
'logger_config' => 'stream',
// maintenance_start (String)
// Start of the window for the daily maintenance cron call.
// The system timezone is used when no timezone is defined here.
'maintenance_start' => '01:00 +00:00',
// maintenance_end (String)
// End of the window for the daily maintenance cron call
// The system timezone is used when no timezone is defined here.
'maintenance_end' => '03:00 +00:00',
// max_batch_queue (Integer)
// Maximum number of batched queue items for a single contact before subsequent messages are discarded.
'max_batch_queue' => 1000,