friendica
/
friendica
mirror of https://github.com/friendica/friendica.git
3
5
Fork 3
Code Issues Releases 13 Wiki Activity
Browse Source

Performance improvements for the poller

pull/2737/head
Michael Vogel 5 years ago
parent
2d7c8c79cf
commit
67b16507f1
4 changed files with 62 additions and 44 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 14
      boot.php
  2. 7
      include/poller.php
  3. 2
      include/post_update.php
  4. 83
      include/queue.php

14
boot.php
View File

@ -1266,8 +1266,20 @@ class App {
function proc_run($args) {
// Add the php path if it is a php call
if (count($args) && ($args[0] === 'php' OR is_int($args[0])))
if (count($args) && ($args[0] === 'php' OR is_int($args[0]))) {
// If the last worker fork was less than 10 seconds before then don't fork another one.
// This should prevent the forking of masses of workers.
if (get_config("system", "worker")) {
if ((time() - get_config("system", "proc_run_started")) < 10)
return;
// Set the timestamp of the last proc_run
set_config("system", "proc_run_started", time());
}
$args[0] = ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php');
}
// add baseurl to args. cli scripts can't construct it
$args[] = $this->get_baseurl();

7
include/poller.php
View File

@ -239,10 +239,13 @@ function poller_kill_stale_workers() {
$max_duration_defaults = array(PRIORITY_SYSTEM => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180);
$max_duration = $max_duration_defaults[$pid["priority"]];
$argv = json_decode($pid["parameter"]);
$argv[0] = basename($argv[0]);
// How long is the process already running?
$duration = (time() - strtotime($pid["executed"])) / 60;
if ($duration > $max_duration) {
logger("Worker process ".$pid["pid"]." (".$pid["parameter"].") took more than ".$max_duration." minutes. It will be killed now.");
logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now.");
posix_kill($pid["pid"], SIGTERM);
// We killed the stale process.
@ -254,7 +257,7 @@ function poller_kill_stale_workers() {
intval(PRIORITY_LOW),
intval($pid["pid"]));
} else
logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
}
}

2
include/post_update.php
View File

@ -159,7 +159,7 @@ function post_update_1198() {
logger("Start", LOGGER_DEBUG);
// Check if the first step is done (Setting "author-id" and "owner-id" in the item table)
$r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 1000");
$r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 100");
if (!$r) {
// Are there unfinished entries in the thread table?
$r = q("SELECT COUNT(*) AS `total` FROM `thread`

83
include/queue.php
View File

@ -45,55 +45,57 @@ function queue_run(&$argv, &$argc){
$deadservers = array();
$serverlist = array();
logger('queue: start');
if (!$queue_id) {
// Handling the pubsubhubbub requests
proc_run(PRIORITY_HIGH,'include/pubsubpublish.php');
logger('queue: start');
$interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval')));
// Handling the pubsubhubbub requests
proc_run(PRIORITY_HIGH,'include/pubsubpublish.php');
// If we are using the worker we don't need a delivery interval
if (get_config("system", "worker"))
$interval = false;
$interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval')));
$r = q("select * from deliverq where 1");
if($r) {
foreach($r as $rr) {
logger('queue: deliverq');
proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']);
if($interval)
// If we are using the worker we don't need a delivery interval
if (get_config("system", "worker"))
$interval = false;
$r = q("select * from deliverq where 1");
if($r) {
foreach($r as $rr) {
logger('queue: deliverq');
proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']);
if($interval)
@time_sleep_until(microtime(true) + (float) $interval);
}
}
}
$r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue`
INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
if($r) {
foreach($r as $rr) {
logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']);
logger('Expired queue data :' . $rr['content'], LOGGER_DATA);
$r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue`
INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
if($r) {
foreach($r as $rr) {
logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']);
logger('Expired queue data :' . $rr['content'], LOGGER_DATA);
}
q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
}
q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
}
if($queue_id) {
$r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id)
);
}
else {
// For the first 12 hours we'll try to deliver every 15 minutes
// After that, we'll only attempt delivery once per hour.
$r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`");
} else {
logger('queue: start for id '.$queue_id);
$r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id)
);
}
if(! $r){
if (!$r){
return;
}
if(! $queue_id)
if (!$queue_id)
call_hooks('queue_predeliver', $a, $r);
@ -107,16 +109,17 @@ function queue_run(&$argv, &$argc){
// queue_predeliver hooks may have changed the queue db details,
// so check again if this entry still needs processing
if($queue_id) {
if($queue_id)
$qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id)
);
}
else {
intval($queue_id));
elseif (get_config("system", "worker")) {
logger('Call queue for id '.$q_item['id']);
proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']);
continue;
} else
$qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ",
intval($q_item['id'])
);
}
intval($q_item['id']));
if(! count($qi))
continue;

Write Preview
Loading…
Cancel
Save