Moved some functionality back to the cron. Speed up things

This commit is contained in:
Michael Vogel 2015-09-11 21:35:58 +02:00
parent ff739b0a23
commit 32e8f3468d
6 changed files with 77 additions and 36 deletions

View File

@ -52,6 +52,20 @@ function cron_run(&$argv, &$argc){
} }
} }
$last = get_config('system','last_cron');
$poll_interval = intval(get_config('system','cron_interval'));
if(! $poll_interval)
$poll_interval = 10;
if($last) {
$next = $last + ($poll_interval * 60);
if($next > time()) {
logger('cron intervall not reached');
return;
}
}
$lockpath = get_lockpath(); $lockpath = get_lockpath();
if ($lockpath != '') { if ($lockpath != '') {
$pidfile = new pidfile($lockpath, 'cron'); $pidfile = new pidfile($lockpath, 'cron');
@ -75,6 +89,22 @@ function cron_run(&$argv, &$argc){
logger('cron: start'); logger('cron: start');
// run queue delivery process in the background
proc_run('php',"include/queue.php");
// run diaspora photo queue process in the background
proc_run('php',"include/dsprphotoq.php");
// run the process to discover global contacts in the background
proc_run('php',"include/discover_poco.php");
// run the process to update locally stored global contacts in the background
proc_run('php',"include/discover_poco.php", "checkcontact");
// expire any expired accounts // expire any expired accounts
q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0
@ -299,6 +329,8 @@ function cron_run(&$argv, &$argc){
logger('cron: end'); logger('cron: end');
set_config('system','last_cron', time());
return; return;
} }

View File

@ -9,13 +9,13 @@ function update_contact($id) {
$r = q("SELECT `url`, `nurl`, `addr`, `alias`, `batch`, `notify`, `poll`, `poco`, `network` FROM `contact` WHERE `id` = %d", intval($id)); $r = q("SELECT `url`, `nurl`, `addr`, `alias`, `batch`, `notify`, `poll`, `poco`, `network` FROM `contact` WHERE `id` = %d", intval($id));
if (!$r) if (!$r)
return; return false;
$ret = probe_url($r[0]["url"]); $ret = probe_url($r[0]["url"]);
// If probe_url fails the network code will be different // If probe_url fails the network code will be different
if ($ret["network"] != $r[0]["network"]) if ($ret["network"] != $r[0]["network"])
return; return false;
$update = false; $update = false;
@ -29,7 +29,7 @@ function update_contact($id) {
} }
if (!$update) if (!$update)
return; return true;
q("UPDATE `contact` SET `url` = '%s', `nurl` = '%s', `addr` = '%s', `alias` = '%s', `batch` = '%s', `notify` = '%s', `poll` = '%s', `poco` = '%s' WHERE `id` = %d", q("UPDATE `contact` SET `url` = '%s', `nurl` = '%s', `addr` = '%s', `alias` = '%s', `batch` = '%s', `notify` = '%s', `poll` = '%s', `poco` = '%s' WHERE `id` = %d",
dbesc($ret['url']), dbesc($ret['url']),
@ -42,6 +42,8 @@ function update_contact($id) {
dbesc($ret['poco']), dbesc($ret['poco']),
intval($id) intval($id)
); );
return true;
} }
// //

View File

@ -168,8 +168,18 @@ function onepoll_run(&$argv, &$argc){
); );
// Update the contact entry // Update the contact entry
if(($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) if(($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) {
update_contact($contact["id"]); if (!poco_reachable($contact['url'])) {
logger("Skipping probably dead contact ".$contact['url']);
return;
}
if (!update_contact($contact["id"])) {
mark_for_death($contact);
return;
} else
unmark_for_death($contact);
}
if($contact['network'] === NETWORK_DFRN) { if($contact['network'] === NETWORK_DFRN) {

View File

@ -26,23 +26,7 @@ function poller_run(&$argv, &$argc){
unset($db_host, $db_user, $db_pass, $db_data); unset($db_host, $db_user, $db_pass, $db_data);
}; };
// run queue delivery process in the background // Run the cron job that calls all other jobs
proc_run('php',"include/queue.php");
// run diaspora photo queue process in the background
proc_run('php',"include/dsprphotoq.php");
// run the process to discover global contacts in the background
proc_run('php',"include/discover_poco.php");
// run the process to update locally stored global contacts in the background
proc_run('php',"include/discover_poco.php", "checkcontact");
// When everything else is done ...
proc_run("php","include/cron.php"); proc_run("php","include/cron.php");
// Cleaning killed processes // Cleaning killed processes
@ -81,7 +65,7 @@ function poller_run(&$argv, &$argc){
if (function_exists($funcname)) { if (function_exists($funcname)) {
logger("Process ".getmypid().": ".$funcname." ".$r[0]["parameter"]); logger("Process ".getmypid().": ".$funcname." ".$r[0]["parameter"]);
$funcname($argv, $argc); $funcname($argv, $argc);
//sleep(10);
logger("Process ".getmypid().": ".$funcname." - done"); logger("Process ".getmypid().": ".$funcname." - done");
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($r[0]["id"])); q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($r[0]["id"]));

View File

@ -22,6 +22,7 @@ function queue_run(&$argv, &$argc){
require_once('include/items.php'); require_once('include/items.php');
require_once('include/bbcode.php'); require_once('include/bbcode.php');
require_once('include/pidfile.php'); require_once('include/pidfile.php');
require_once('include/socgraph.php');
load_config('config'); load_config('config');
load_config('system'); load_config('system');
@ -88,7 +89,7 @@ function queue_run(&$argv, &$argc){
else { else {
// For the first 12 hours we'll try to deliver every 15 minutes // For the first 12 hours we'll try to deliver every 15 minutes
// After that, we'll only attempt delivery once per hour. // After that, we'll only attempt delivery once per hour.
$r = q("SELECT `id` FROM `queue` WHERE (( `created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ) OR ( `last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR ))"); $r = q("SELECT `id` FROM `queue` WHERE (( `created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ) OR ( `last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR ))");
} }
@ -107,7 +108,7 @@ function queue_run(&$argv, &$argc){
foreach($r as $q_item) { foreach($r as $q_item) {
// queue_predeliver hooks may have changed the queue db details, // queue_predeliver hooks may have changed the queue db details,
// so check again if this entry still needs processing // so check again if this entry still needs processing
if($queue_id) { if($queue_id) {
@ -132,12 +133,18 @@ function queue_run(&$argv, &$argc){
continue; continue;
} }
if(in_array($c[0]['notify'],$deadguys)) { if(in_array($c[0]['notify'],$deadguys)) {
logger('queue: skipping known dead url: ' . $c[0]['notify']); logger('queue: skipping known dead url: ' . $c[0]['notify']);
update_queue_time($q_item['id']); update_queue_time($q_item['id']);
continue; continue;
} }
$u = q("SELECT `user`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey` if (!poco_reachable($c[0]['url'])) {
logger('queue: skipping probably dead url: ' . $c[0]['url']);
update_queue_time($q_item['id']);
continue;
}
$u = q("SELECT `user`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey`
FROM `user` WHERE `uid` = %d LIMIT 1", FROM `user` WHERE `uid` = %d LIMIT 1",
intval($c[0]['uid']) intval($c[0]['uid'])
); );
@ -194,9 +201,9 @@ function queue_run(&$argv, &$argc){
call_hooks('queue_deliver', $a, $params); call_hooks('queue_deliver', $a, $params);
if($params['result']) if($params['result'])
remove_queue_item($q_item['id']); remove_queue_item($q_item['id']);
else else
update_queue_time($q_item['id']); update_queue_time($q_item['id']);
break; break;

View File

@ -234,7 +234,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
} }
if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "") OR $alternate) if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "") OR $alternate)
AND poco_reachable($profile_url, $server_url, $network, true)) { AND poco_reachable($profile_url, $server_url, $network, false)) {
$data = probe_url($profile_url); $data = probe_url($profile_url);
$orig_profile = $profile_url; $orig_profile = $profile_url;
@ -1296,8 +1296,11 @@ function poco_discover($complete = false) {
if ($r) if ($r)
foreach ($r AS $server) { foreach ($r AS $server) {
if (!poco_check_server($server["url"], $server["network"])) if (!poco_check_server($server["url"], $server["network"])) {
// The server is not reachable? Okay, then we will try it later
q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
continue; continue;
}
// Fetch all users from the other server // Fetch all users from the other server
$url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; $url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
@ -1338,10 +1341,13 @@ function poco_discover($complete = false) {
q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
if (!$complete AND (--$no_of_queries == 0)) if (!$complete AND (--$no_of_queries == 0))
break; break;
// If the server hadn't replied correctly, then force a sanity check } else {
} elseif (!poco_check_server($server["url"], $server["network"], true)) // If the server hadn't replied correctly, then force a sanity check
q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); poco_check_server($server["url"], $server["network"], true);
// If we couldn't reach the server, we will try it some time later
q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
}
} }
} }