tobias/friendica
1
0
Fork 0
Code Pull requests Activity

Merge pull request #2737 from annando/1608-worker-again

Browse source 
Performance improvements for the poller
This commit is contained in:
Tobias Diekershoff 2016-08-14 22:10:13 +02:00 committed by GitHub
commit 789e9577c6
4 changed files with 62 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
boot.php
View file

@ -1266,8 +1266,20 @@ class App {
function proc_run($args) { function proc_run($args) {
// Add the php path if it is a php call // Add the php path if it is a php call
if (count($args) && ($args[0] === 'php' OR is_int($args[0]))) if (count($args) && ($args[0] === 'php' OR is_int($args[0]))) {
// If the last worker fork was less than 10 seconds before then don't fork another one.
// This should prevent the forking of masses of workers.
if (get_config("system", "worker")) {
if ((time() - get_config("system", "proc_run_started")) < 10)
return;
// Set the timestamp of the last proc_run
set_config("system", "proc_run_started", time());
}
$args[0] = ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php'); $args[0] = ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php');
}
// add baseurl to args. cli scripts can't construct it // add baseurl to args. cli scripts can't construct it
$args[] = $this->get_baseurl(); $args[] = $this->get_baseurl();

7
include/poller.php
View file

@ -239,10 +239,13 @@ function poller_kill_stale_workers() {
$max_duration_defaults = array(PRIORITY_SYSTEM => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180); $max_duration_defaults = array(PRIORITY_SYSTEM => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180);
$max_duration = $max_duration_defaults[$pid["priority"]]; $max_duration = $max_duration_defaults[$pid["priority"]];
$argv = json_decode($pid["parameter"]);
$argv[0] = basename($argv[0]);
// How long is the process already running? // How long is the process already running?
$duration = (time() - strtotime($pid["executed"])) / 60; $duration = (time() - strtotime($pid["executed"])) / 60;
if ($duration > $max_duration) { if ($duration > $max_duration) {
logger("Worker process ".$pid["pid"]." (".$pid["parameter"].") took more than ".$max_duration." minutes. It will be killed now."); logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now.");
posix_kill($pid["pid"], SIGTERM); posix_kill($pid["pid"], SIGTERM);
// We killed the stale process. // We killed the stale process.
@ -254,7 +257,7 @@ function poller_kill_stale_workers() {
intval(PRIORITY_LOW), intval(PRIORITY_LOW),
intval($pid["pid"])); intval($pid["pid"]));
} else } else
logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG); logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
} }
} }

2
include/post_update.php
View file

@ -159,7 +159,7 @@ function post_update_1198() {
logger("Start", LOGGER_DEBUG); logger("Start", LOGGER_DEBUG);
// Check if the first step is done (Setting "author-id" and "owner-id" in the item table) // Check if the first step is done (Setting "author-id" and "owner-id" in the item table)
$r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 1000"); $r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 100");
if (!$r) { if (!$r) {
// Are there unfinished entries in the thread table? // Are there unfinished entries in the thread table?
$r = q("SELECT COUNT(*) AS `total` FROM `thread` $r = q("SELECT COUNT(*) AS `total` FROM `thread`

83
include/queue.php
View file

@ -45,55 +45,57 @@ function queue_run(&$argv, &$argc){
$deadservers = array(); $deadservers = array();
$serverlist = array(); $serverlist = array();
logger('queue: start'); if (!$queue_id) {
// Handling the pubsubhubbub requests logger('queue: start');
proc_run(PRIORITY_HIGH,'include/pubsubpublish.php');
$interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval'))); // Handling the pubsubhubbub requests
proc_run(PRIORITY_HIGH,'include/pubsubpublish.php');
// If we are using the worker we don't need a delivery interval $interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval')));
if (get_config("system", "worker"))
$interval = false;
$r = q("select * from deliverq where 1"); // If we are using the worker we don't need a delivery interval
if($r) { if (get_config("system", "worker"))
foreach($r as $rr) { $interval = false;
logger('queue: deliverq');
proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']); $r = q("select * from deliverq where 1");
if($interval) if($r) {
foreach($r as $rr) {
logger('queue: deliverq');
proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']);
if($interval)
@time_sleep_until(microtime(true) + (float) $interval); @time_sleep_until(microtime(true) + (float) $interval);
}
} }
}
$r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue` $r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue`
INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id` INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
if($r) { if($r) {
foreach($r as $rr) { foreach($r as $rr) {
logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']); logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']);
logger('Expired queue data :' . $rr['content'], LOGGER_DATA); logger('Expired queue data :' . $rr['content'], LOGGER_DATA);
}
q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
} }
q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
}
if($queue_id) {
$r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id)
);
}
else {
// For the first 12 hours we'll try to deliver every 15 minutes // For the first 12 hours we'll try to deliver every 15 minutes
// After that, we'll only attempt delivery once per hour. // After that, we'll only attempt delivery once per hour.
$r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`"); $r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`");
} else {
logger('queue: start for id '.$queue_id);
$r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id)
);
} }
if(! $r){
if (!$r){
return; return;
} }
if(! $queue_id) if (!$queue_id)
call_hooks('queue_predeliver', $a, $r); call_hooks('queue_predeliver', $a, $r);
@ -107,16 +109,17 @@ function queue_run(&$argv, &$argc){
// queue_predeliver hooks may have changed the queue db details, // queue_predeliver hooks may have changed the queue db details,
// so check again if this entry still needs processing // so check again if this entry still needs processing
if($queue_id) { if($queue_id)
$qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1", $qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id) intval($queue_id));
); elseif (get_config("system", "worker")) {
} logger('Call queue for id '.$q_item['id']);
else { proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']);
continue;
} else
$qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ", $qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ",
intval($q_item['id']) intval($q_item['id']));
);
}
if(! count($qi)) if(! count($qi))
continue; continue;