From 67b16507f1747e37b274f170313943b526dd5090 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 14 Aug 2016 21:02:29 +0200 Subject: [PATCH] Performance improvements for the poller --- boot.php | 14 ++++++- include/poller.php | 7 +++- include/post_update.php | 2 +- include/queue.php | 83 +++++++++++++++++++++-------------------- 4 files changed, 62 insertions(+), 44 deletions(-) diff --git a/boot.php b/boot.php index 679116212c..208293a925 100644 --- a/boot.php +++ b/boot.php @@ -1266,8 +1266,20 @@ class App { function proc_run($args) { // Add the php path if it is a php call - if (count($args) && ($args[0] === 'php' OR is_int($args[0]))) + if (count($args) && ($args[0] === 'php' OR is_int($args[0]))) { + + // If the last worker fork was less than 10 seconds before then don't fork another one. + // This should prevent the forking of masses of workers. + if (get_config("system", "worker")) { + if ((time() - get_config("system", "proc_run_started")) < 10) + return; + + // Set the timestamp of the last proc_run + set_config("system", "proc_run_started", time()); + } + $args[0] = ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php'); + } // add baseurl to args. cli scripts can't construct it $args[] = $this->get_baseurl(); diff --git a/include/poller.php b/include/poller.php index ecdb9eb0d3..1f4d25732e 100644 --- a/include/poller.php +++ b/include/poller.php @@ -239,10 +239,13 @@ function poller_kill_stale_workers() { $max_duration_defaults = array(PRIORITY_SYSTEM => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180); $max_duration = $max_duration_defaults[$pid["priority"]]; + $argv = json_decode($pid["parameter"]); + $argv[0] = basename($argv[0]); + // How long is the process already running? $duration = (time() - strtotime($pid["executed"])) / 60; if ($duration > $max_duration) { - logger("Worker process ".$pid["pid"]." (".$pid["parameter"].") took more than ".$max_duration." minutes. It will be killed now."); + logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now."); posix_kill($pid["pid"], SIGTERM); // We killed the stale process. @@ -254,7 +257,7 @@ function poller_kill_stale_workers() { intval(PRIORITY_LOW), intval($pid["pid"])); } else - logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG); + logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG); } } diff --git a/include/post_update.php b/include/post_update.php index 6918bf0fa4..d45a51b5f2 100644 --- a/include/post_update.php +++ b/include/post_update.php @@ -159,7 +159,7 @@ function post_update_1198() { logger("Start", LOGGER_DEBUG); // Check if the first step is done (Setting "author-id" and "owner-id" in the item table) - $r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 1000"); + $r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 100"); if (!$r) { // Are there unfinished entries in the thread table? $r = q("SELECT COUNT(*) AS `total` FROM `thread` diff --git a/include/queue.php b/include/queue.php index 9779a0733d..47a3596bdb 100644 --- a/include/queue.php +++ b/include/queue.php @@ -45,55 +45,57 @@ function queue_run(&$argv, &$argc){ $deadservers = array(); $serverlist = array(); - logger('queue: start'); + if (!$queue_id) { - // Handling the pubsubhubbub requests - proc_run(PRIORITY_HIGH,'include/pubsubpublish.php'); + logger('queue: start'); - $interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval'))); + // Handling the pubsubhubbub requests + proc_run(PRIORITY_HIGH,'include/pubsubpublish.php'); - // If we are using the worker we don't need a delivery interval - if (get_config("system", "worker")) - $interval = false; + $interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval'))); - $r = q("select * from deliverq where 1"); - if($r) { - foreach($r as $rr) { - logger('queue: deliverq'); - proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']); - if($interval) + // If we are using the worker we don't need a delivery interval + if (get_config("system", "worker")) + $interval = false; + + $r = q("select * from deliverq where 1"); + if($r) { + foreach($r as $rr) { + logger('queue: deliverq'); + proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']); + if($interval) @time_sleep_until(microtime(true) + (float) $interval); + } } - } - $r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue` - INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id` - WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); - if($r) { - foreach($r as $rr) { - logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']); - logger('Expired queue data :' . $rr['content'], LOGGER_DATA); + $r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue` + INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id` + WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); + if($r) { + foreach($r as $rr) { + logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']); + logger('Expired queue data :' . $rr['content'], LOGGER_DATA); + } + q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); } - q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); - } - - if($queue_id) { - $r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1", - intval($queue_id) - ); - } - else { // For the first 12 hours we'll try to deliver every 15 minutes // After that, we'll only attempt delivery once per hour. $r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`"); + } else { + logger('queue: start for id '.$queue_id); + + $r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1", + intval($queue_id) + ); } - if(! $r){ + + if (!$r){ return; } - if(! $queue_id) + if (!$queue_id) call_hooks('queue_predeliver', $a, $r); @@ -107,16 +109,17 @@ function queue_run(&$argv, &$argc){ // queue_predeliver hooks may have changed the queue db details, // so check again if this entry still needs processing - if($queue_id) { + if($queue_id) $qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1", - intval($queue_id) - ); - } - else { + intval($queue_id)); + elseif (get_config("system", "worker")) { + logger('Call queue for id '.$q_item['id']); + proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']); + continue; + } else $qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ", - intval($q_item['id']) - ); - } + intval($q_item['id'])); + if(! count($qi)) continue;