From d673f44c5bf4f57c89767f26077ce2092e12b9c5 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 3 Aug 2016 10:03:05 +0200 Subject: [PATCH] Split cron jobs in cronjobs, introduce fastlane for high priority tasks --- include/cron.php | 82 +++++++++++++++++++++++++++----------------- include/cronjobs.php | 81 +++++++++++++++++++++++++++++++++++++++++++ include/poller.php | 18 ++++++++++ 3 files changed, 149 insertions(+), 32 deletions(-) create mode 100644 include/cronjobs.php diff --git a/include/cron.php b/include/cron.php index 82c8cc7a16..6afbeca1bd 100644 --- a/include/cron.php +++ b/include/cron.php @@ -80,43 +80,38 @@ function cron_run(&$argv, &$argc){ proc_run(PRIORITY_LOW,"include/discover_poco.php", "checkcontact"); - // expire any expired accounts + // Expire and remove user entries + cron_expire_and_remove_users(); - q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 - AND `account_expires_on` != '0000-00-00 00:00:00' - AND `account_expires_on` < UTC_TIMESTAMP() "); + // If the worker is active, split the jobs in several sub processes + if (get_config("system", "worker")) { + // Check OStatus conversations + proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "ostatus_mentions"); - // delete user and contact records for recently removed accounts + // Check every conversation + proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "ostatus_conversations"); - $r = q("SELECT * FROM `user` WHERE `account_removed` = 1 AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); - if ($r) { - foreach($r as $user) { - q("DELETE FROM `contact` WHERE `uid` = %d", intval($user['uid'])); - q("DELETE FROM `user` WHERE `uid` = %d", intval($user['uid'])); - } + // Call possible post update functions + proc_run(PRIORITY_LOW, "include/cronjobs.php", "post_update"); + + // update nodeinfo data + proc_run(PRIORITY_LOW, "include/cronjobs.php", "nodeinfo"); + } else { + // Check OStatus conversations + // Check only conversations with mentions (for a longer time) + ostatus::check_conversations(true); + + // Check every conversation + ostatus::check_conversations(false); + + // Call possible post update functions + // see include/post_update.php for more details + post_update(); + + // update nodeinfo data + nodeinfo_cron(); } - $abandon_days = intval(get_config('system','account_abandon_days')); - if($abandon_days < 1) - $abandon_days = 0; - - // Check OStatus conversations - // Check only conversations with mentions (for a longer time) - ostatus::check_conversations(true); - - // Check every conversation - ostatus::check_conversations(false); - - // Call possible post update functions - // see include/post_update.php for more details - post_update(); - - // update nodeinfo data - nodeinfo_cron(); - - /// @TODO Regenerate usage statistics - // q("ANALYZE TABLE `item`"); - // once daily run birthday_updates and then expire in background $d1 = get_config('system','last_expire_day'); @@ -152,6 +147,25 @@ function cron_run(&$argv, &$argc){ return; } +/** + * @brief Expire and remove user entries + */ +function cron_expire_and_remove_users() { + // expire any expired accounts + q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 + AND `account_expires_on` != '0000-00-00 00:00:00' + AND `account_expires_on` < UTC_TIMESTAMP() "); + + // delete user and contact records for recently removed accounts + $r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); + if ($r) { + foreach($r as $user) { + q("DELETE FROM `contact` WHERE `uid` = %d", intval($user['uid'])); + q("DELETE FROM `user` WHERE `uid` = %d", intval($user['uid'])); + } + } +} + /** * @brief Poll contacts for unreceived messages * @@ -197,6 +211,10 @@ function cron_poll_contacts($argc, $argv) { // and which have a polling address and ignore Diaspora since // we are unable to match those posts with a Diaspora GUID and prevent duplicates. + $abandon_days = intval(get_config('system','account_abandon_days')); + if($abandon_days < 1) + $abandon_days = 0; + $abandon_sql = (($abandon_days) ? sprintf(" AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL %d DAY ", intval($abandon_days)) : '' diff --git a/include/cronjobs.php b/include/cronjobs.php new file mode 100644 index 0000000000..ed27e23293 --- /dev/null +++ b/include/cronjobs.php @@ -0,0 +1,81 @@ +set_baseurl(get_config('system','url')); + + // No parameter set? So return + if ($argc <= 1) + return; + + // Check OStatus conversations + // Check only conversations with mentions (for a longer time) + if ($argv[1] == 'ostatus_mentions') { + ostatus::check_conversations(true); + return; + } + + // Check every conversation + if ($argv[1] == 'ostatus_conversations') { + ostatus::check_conversations(false); + return; + } + + // Call possible post update functions + // see include/post_update.php for more details + if ($argv[1] == 'post_update') { + post_update(); + return; + } + + // update nodeinfo data + if ($argv[1] == 'nodeinfo') { + nodeinfo_cron(); + return; + } + + return; +} + +if (array_search(__file__,get_included_files())===0){ + cronjobs_run($_SERVER["argv"],$_SERVER["argc"]); + killme(); +} diff --git a/include/poller.php b/include/poller.php index 811b81e174..622aa83bd7 100644 --- a/include/poller.php +++ b/include/poller.php @@ -270,6 +270,24 @@ function poller_too_much_workers() { $slope = $maxworkers / pow($maxsysload, $exponent); $queues = ceil($slope * pow(max(0, $maxsysload - $load), $exponent)); + if (Config::get("system", "worker_fastlane", false) AND ($queues > 0) AND ($active >= $queues)) { + $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `priority` = %d AND `executed` = '0000-00-00 00:00:00'", + intval(PRIORITY_HIGH)); + $high_waiting = $s[0]["total"]; + + $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `priority` = %d AND `executed` != '0000-00-00 00:00:00'", + intval(PRIORITY_HIGH)); + $high_running = $s[0]["total"]; + + logger("High waiting: ".$high_waiting." - high running: ".$high_running); + + /// @todo define maximum number of fastlanes + if (($high_waiting > 0) AND ($high_running == 0)) { + logger("There are ".$high_waiting." high priority jobs waiting but none is executed. Open a fastlane.", LOGGER_DEBUG); + $queue = $active + 1; + } + } + $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00'"); $entries = $s[0]["total"];