Browse Source

Merge pull request #9434 from annando/dbclean-limit

Post expiry: Limit for deletions
pull/9405/head
Hypolite Petovan 1 week ago
committed by GitHub
parent
commit
b3e5621d37
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 103 additions and 24 deletions
  1. +44
    -0
      src/Core/Worker.php
  2. +2
    -2
      src/Worker/Cron.php
  3. +45
    -21
      src/Worker/ExpirePosts.php
  4. +12
    -1
      static/defaults.config.php

+ 44
- 0
src/Core/Worker.php View File

@@ -1394,4 +1394,48 @@ class Worker

return (bool)$row['jobs'];
}

/**
* Check if the system is inside the defined maintenance window
*
* @return boolean
*/
public static function isInMaintenanceWindow(bool $check_last_execution = false)
{
// Calculate the seconds of the start end end of the maintenance window
$start = strtotime(DI::config()->get('system', 'maintenance_start')) % 86400;
$end = strtotime(DI::config()->get('system', 'maintenance_end')) % 86400;

Logger::info('Maintenance window', ['start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);

if ($check_last_execution) {
// Calculate the window duration
$duration = max($start, $end) - min($start, $end);

// Quit when the last cron execution had been after the previous window
$last_cron = DI::config()->get('system', 'last_cron_daily');
if ($last_cron + $duration > time()) {
Logger::info('The Daily cron had been executed recently', ['last' => date(DateTimeFormat::MYSQL, $last_cron), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
return false;
}
}

$current = time() % 86400;

if ($start < $end) {
// Execute if we are inside the window
$execute = ($current >= $start) && ($current <= $end);
} else {
// Don't execute if we are outside the window
$execute = !(($current > $end) && ($current < $start));
}

if ($execute) {
Logger::info('We are inside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
} else {
Logger::info('We are outside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
}
return $execute;
}
}

+ 2
- 2
src/Worker/Cron.php View File

@@ -93,8 +93,8 @@ class Cron
DI::config()->set('system', 'last_cron_hourly', time());
}

// Daily cron calls
if (DI::config()->get('system', 'last_cron_daily', 0) + 86400 < time()) {
// Daily maintenance cron calls
if (Worker::isInMaintenanceWindow(true)) {

Worker::add(PRIORITY_LOW, 'UpdateContactBirthdays');



+ 45
- 21
src/Worker/ExpirePosts.php View File

@@ -22,6 +22,7 @@
namespace Friendica\Worker;

use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\DateTimeFormat;
@@ -39,35 +40,58 @@ class ExpirePosts
$expire_days_unclaimed = $expire_days;
}

$limit = DI::config()->get('system', 'dbclean-expire-limit');
if (empty($limit)) {
return;
}

if (!empty($expire_days)) {
Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `thread`
INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts`
WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin`
AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item`
WHERE (`item`.`starred` OR (`item`.`resource-id` != '')
OR (`item`.`event-id` != '') OR (`item`.`attach` != '')
OR `item`.`wall` OR `item`.`origin`
OR `uri-id` IN (SELECT `uri-id` FROM `post-category`
WHERE `uri-id` = `item`.`uri-id`))
AND `item`.`parent` = `thread`.`iid`))", $expire_days);
do {
Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `thread`
INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts`
WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin`
AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item`
WHERE (`item`.`starred` OR (`item`.`resource-id` != '')
OR (`item`.`event-id` != '') OR (`item`.`attach` != '')
OR `item`.`wall` OR `item`.`origin`
OR `uri-id` IN (SELECT `uri-id` FROM `post-category`
WHERE `uri-id` = `item`.`uri-id`))
AND `item`.`parent` = `thread`.`iid`))
ORDER BY `id` LIMIT ?", $expire_days, $limit);

Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => DBA::affectedRows()]);
$rows = DBA::affectedRows();
Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => $rows]);

if (!Worker::isInMaintenanceWindow()) {
Logger::notice('We are outside of the maintenance window, quitting');
return;
}
} while ($rows >= $limit);
}

if (!empty($expire_days_unclaimed)) {
$expiry_date = DateTimeFormat::utc('now - ' . $expire_days_unclaimed . ' days', DateTimeFormat::MYSQL);

Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?))",
GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date);
do {
Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]);
$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
(SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?)
AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?))
ORDER BY `id` LIMIT ?",
GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date, $limit);

$rows = DBA::affectedRows();
Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => $rows]);

Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => DBA::affectedRows()]);
if (!Worker::isInMaintenanceWindow()) {
Logger::notice('We are outside of the maintenance window, quitting');
return;
}
} while ($rows >= $limit);
}
}
}

+ 12
- 1
static/defaults.config.php View File

@@ -187,7 +187,8 @@ return [

// dbclean-expire-limit (Integer)
// This defines the number of items that are to be deleted in a single call.
// Reduce this value when you are getting memory issues.
// Reduce this value when you are getting lock issues.
// A value of 0 disables the deletion process.
'dbclean-expire-limit' => 1000,

// diaspora_test (Boolean)
@@ -299,6 +300,16 @@ return [
// Sets the logging adapter of Friendica globally (monolog, syslog, stream)
'logger_config' => 'stream',

// maintenance_start (String)
// Start of the window for the daily maintenance cron call.
// The system timezone is used when no timezone is defined here.
'maintenance_start' => '01:00 +00:00',

// maintenance_end (String)
// End of the window for the daily maintenance cron call
// The system timezone is used when no timezone is defined here.
'maintenance_end' => '03:00 +00:00',

// max_batch_queue (Integer)
// Maximum number of batched queue items for a single contact before subsequent messages are discarded.
'max_batch_queue' => 1000,


Loading…
Cancel
Save