From 32e8f3468d17f9f84b308aa903f7efa1fa22441f Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 11 Sep 2015 21:35:58 +0200 Subject: [PATCH] Moved some functionality back to the cron. Speed up things --- include/cron.php | 32 ++++++++++++++++++++++++++++++++ include/follow.php | 8 +++++--- include/onepoll.php | 14 ++++++++++++-- include/poller.php | 20 ++------------------ include/queue.php | 23 +++++++++++++++-------- include/socgraph.php | 16 +++++++++++----- 6 files changed, 77 insertions(+), 36 deletions(-) diff --git a/include/cron.php b/include/cron.php index ea7fd2c606..712befb1ff 100644 --- a/include/cron.php +++ b/include/cron.php @@ -52,6 +52,20 @@ function cron_run(&$argv, &$argc){ } } + $last = get_config('system','last_cron'); + + $poll_interval = intval(get_config('system','cron_interval')); + if(! $poll_interval) + $poll_interval = 10; + + if($last) { + $next = $last + ($poll_interval * 60); + if($next > time()) { + logger('cron intervall not reached'); + return; + } + } + $lockpath = get_lockpath(); if ($lockpath != '') { $pidfile = new pidfile($lockpath, 'cron'); @@ -75,6 +89,22 @@ function cron_run(&$argv, &$argc){ logger('cron: start'); + // run queue delivery process in the background + + proc_run('php',"include/queue.php"); + + // run diaspora photo queue process in the background + + proc_run('php',"include/dsprphotoq.php"); + + // run the process to discover global contacts in the background + + proc_run('php',"include/discover_poco.php"); + + // run the process to update locally stored global contacts in the background + + proc_run('php',"include/discover_poco.php", "checkcontact"); + // expire any expired accounts q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 @@ -299,6 +329,8 @@ function cron_run(&$argv, &$argc){ logger('cron: end'); + set_config('system','last_cron', time()); + return; } diff --git a/include/follow.php b/include/follow.php index 217b9d07b7..ca0228cc0f 100644 --- a/include/follow.php +++ b/include/follow.php @@ -9,13 +9,13 @@ function update_contact($id) { $r = q("SELECT `url`, `nurl`, `addr`, `alias`, `batch`, `notify`, `poll`, `poco`, `network` FROM `contact` WHERE `id` = %d", intval($id)); if (!$r) - return; + return false; $ret = probe_url($r[0]["url"]); // If probe_url fails the network code will be different if ($ret["network"] != $r[0]["network"]) - return; + return false; $update = false; @@ -29,7 +29,7 @@ function update_contact($id) { } if (!$update) - return; + return true; q("UPDATE `contact` SET `url` = '%s', `nurl` = '%s', `addr` = '%s', `alias` = '%s', `batch` = '%s', `notify` = '%s', `poll` = '%s', `poco` = '%s' WHERE `id` = %d", dbesc($ret['url']), @@ -42,6 +42,8 @@ function update_contact($id) { dbesc($ret['poco']), intval($id) ); + + return true; } // diff --git a/include/onepoll.php b/include/onepoll.php index e8fc97b21e..0e58a776ca 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -168,8 +168,18 @@ function onepoll_run(&$argv, &$argc){ ); // Update the contact entry - if(($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) - update_contact($contact["id"]); + if(($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) { + if (!poco_reachable($contact['url'])) { + logger("Skipping probably dead contact ".$contact['url']); + return; + } + + if (!update_contact($contact["id"])) { + mark_for_death($contact); + return; + } else + unmark_for_death($contact); + } if($contact['network'] === NETWORK_DFRN) { diff --git a/include/poller.php b/include/poller.php index 05e936936c..74d23a5482 100644 --- a/include/poller.php +++ b/include/poller.php @@ -26,23 +26,7 @@ function poller_run(&$argv, &$argc){ unset($db_host, $db_user, $db_pass, $db_data); }; - // run queue delivery process in the background - - proc_run('php',"include/queue.php"); - - // run diaspora photo queue process in the background - - proc_run('php',"include/dsprphotoq.php"); - - // run the process to discover global contacts in the background - - proc_run('php',"include/discover_poco.php"); - - // run the process to update locally stored global contacts in the background - - proc_run('php',"include/discover_poco.php", "checkcontact"); - - // When everything else is done ... + // Run the cron job that calls all other jobs proc_run("php","include/cron.php"); // Cleaning killed processes @@ -81,7 +65,7 @@ function poller_run(&$argv, &$argc){ if (function_exists($funcname)) { logger("Process ".getmypid().": ".$funcname." ".$r[0]["parameter"]); $funcname($argv, $argc); - //sleep(10); + logger("Process ".getmypid().": ".$funcname." - done"); q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($r[0]["id"])); diff --git a/include/queue.php b/include/queue.php index 0edd64fdb1..3f6686ec6e 100644 --- a/include/queue.php +++ b/include/queue.php @@ -22,6 +22,7 @@ function queue_run(&$argv, &$argc){ require_once('include/items.php'); require_once('include/bbcode.php'); require_once('include/pidfile.php'); + require_once('include/socgraph.php'); load_config('config'); load_config('system'); @@ -88,7 +89,7 @@ function queue_run(&$argv, &$argc){ else { // For the first 12 hours we'll try to deliver every 15 minutes - // After that, we'll only attempt delivery once per hour. + // After that, we'll only attempt delivery once per hour. $r = q("SELECT `id` FROM `queue` WHERE (( `created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ) OR ( `last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR ))"); } @@ -107,7 +108,7 @@ function queue_run(&$argv, &$argc){ foreach($r as $q_item) { - // queue_predeliver hooks may have changed the queue db details, + // queue_predeliver hooks may have changed the queue db details, // so check again if this entry still needs processing if($queue_id) { @@ -132,12 +133,18 @@ function queue_run(&$argv, &$argc){ continue; } if(in_array($c[0]['notify'],$deadguys)) { - logger('queue: skipping known dead url: ' . $c[0]['notify']); - update_queue_time($q_item['id']); - continue; + logger('queue: skipping known dead url: ' . $c[0]['notify']); + update_queue_time($q_item['id']); + continue; } - $u = q("SELECT `user`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey` + if (!poco_reachable($c[0]['url'])) { + logger('queue: skipping probably dead url: ' . $c[0]['url']); + update_queue_time($q_item['id']); + continue; + } + + $u = q("SELECT `user`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey` FROM `user` WHERE `uid` = %d LIMIT 1", intval($c[0]['uid']) ); @@ -194,9 +201,9 @@ function queue_run(&$argv, &$argc){ call_hooks('queue_deliver', $a, $params); if($params['result']) - remove_queue_item($q_item['id']); + remove_queue_item($q_item['id']); else - update_queue_time($q_item['id']); + update_queue_time($q_item['id']); break; diff --git a/include/socgraph.php b/include/socgraph.php index 6e2b6ea158..1a92a72566 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -234,7 +234,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca } if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "") OR $alternate) - AND poco_reachable($profile_url, $server_url, $network, true)) { + AND poco_reachable($profile_url, $server_url, $network, false)) { $data = probe_url($profile_url); $orig_profile = $profile_url; @@ -1296,8 +1296,11 @@ function poco_discover($complete = false) { if ($r) foreach ($r AS $server) { - if (!poco_check_server($server["url"], $server["network"])) + if (!poco_check_server($server["url"], $server["network"])) { + // The server is not reachable? Okay, then we will try it later + q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); continue; + } // Fetch all users from the other server $url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; @@ -1338,10 +1341,13 @@ function poco_discover($complete = false) { q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); if (!$complete AND (--$no_of_queries == 0)) break; - // If the server hadn't replied correctly, then force a sanity check - } elseif (!poco_check_server($server["url"], $server["network"], true)) - q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); + } else { + // If the server hadn't replied correctly, then force a sanity check + poco_check_server($server["url"], $server["network"], true); + // If we couldn't reach the server, we will try it some time later + q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); + } } }