From ec9c9f0be78f9db691ee1c7174f10a7850033717 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 8 Mar 2016 00:20:06 +0100 Subject: [PATCH 1/5] Don't create lock files if the process is called from the poller via the worker --- include/cron.php | 51 +++++++++++++++++++------------------ include/cronhooks.php | 49 ++++++++++++++++++----------------- include/discover_poco.php | 46 ++++++++++++++++++--------------- include/onepoll.php | 21 ++++++++------- include/ostatus.php | 16 ++++++++---- include/pubsubpublish.php | 25 ++++++++++-------- include/queue.php | 25 ++++++++++-------- include/update_gcontact.php | 21 ++++++++------- 8 files changed, 141 insertions(+), 113 deletions(-) diff --git a/include/cron.php b/include/cron.php index 3acf711dd..614328171 100644 --- a/include/cron.php +++ b/include/cron.php @@ -40,15 +40,33 @@ function cron_run(&$argv, &$argc){ load_config('config'); load_config('system'); - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; + // Don't check this stuff if the function is called by the poller + if (App::callstack() != "poller_run") { + $maxsysload = intval(get_config('system','maxloadavg')); + if($maxsysload < 1) + $maxsysload = 50; - $load = current_load(); - if($load) { - if(intval($load) > $maxsysload) { - logger('system: load ' . $load . ' too high. cron deferred to next scheduled run.'); - return; + $load = current_load(); + if($load) { + if(intval($load) > $maxsysload) { + logger('system: load '.$load.' too high. cron deferred to next scheduled run.'); + return; + } + } + + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'cron'); + if($pidfile->is_already_running()) { + logger("cron: Already running"); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("cron: killed stale process"); + // Calling a new instance + proc_run('php','include/cron.php'); + } + exit; + } } } @@ -66,23 +84,6 @@ function cron_run(&$argv, &$argc){ } } - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'cron'); - if($pidfile->is_already_running()) { - logger("cron: Already running"); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("cron: killed stale process"); - // Calling a new instance - proc_run('php','include/cron.php'); - } - exit; - } - } - - - $a->set_baseurl(get_config('system','url')); load_hooks(); diff --git a/include/cronhooks.php b/include/cronhooks.php index 8c70008e4..71cb0fb7b 100644 --- a/include/cronhooks.php +++ b/include/cronhooks.php @@ -24,15 +24,33 @@ function cronhooks_run(&$argv, &$argc){ load_config('config'); load_config('system'); - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; + // Don't check this stuff if the function is called by the poller + if (App::callstack() != "poller_run") { + $maxsysload = intval(get_config('system','maxloadavg')); + if($maxsysload < 1) + $maxsysload = 50; - $load = current_load(); - if($load) { - if(intval($load) > $maxsysload) { - logger('system: load ' . $load . ' too high. Cronhooks deferred to next scheduled run.'); - return; + $load = current_load(); + if($load) { + if(intval($load) > $maxsysload) { + logger('system: load ' . $load . ' too high. Cronhooks deferred to next scheduled run.'); + return; + } + } + + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'cronhooks'); + if($pidfile->is_already_running()) { + logger("cronhooks: Already running"); + if ($pidfile->running_time() > 19*60) { + $pidfile->kill(); + logger("cronhooks: killed stale process"); + // Calling a new instance + proc_run('php','include/cronhooks.php'); + } + exit; + } } } @@ -50,21 +68,6 @@ function cronhooks_run(&$argv, &$argc){ } } - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'cronhooks'); - if($pidfile->is_already_running()) { - logger("cronhooks: Already running"); - if ($pidfile->running_time() > 19*60) { - $pidfile->kill(); - logger("cronhooks: killed stale process"); - // Calling a new instance - proc_run('php','include/cronhooks.php'); - } - exit; - } - } - $a->set_baseurl(get_config('system','url')); load_hooks(); diff --git a/include/discover_poco.php b/include/discover_poco.php index a8f670334..550c9897b 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -25,15 +25,18 @@ function discover_poco_run(&$argv, &$argc){ load_config('config'); load_config('system'); - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; + // Don't check this stuff if the function is called by the poller + if (App::callstack() != "poller_run") { + $maxsysload = intval(get_config('system','maxloadavg')); + if($maxsysload < 1) + $maxsysload = 50; - $load = current_load(); - if($load) { - if(intval($load) > $maxsysload) { - logger('system: load ' . $load . ' too high. discover_poco deferred to next scheduled run.'); - return; + $load = current_load(); + if($load) { + if(intval($load) > $maxsysload) { + logger('system: load '.$load.' too high. discover_poco deferred to next scheduled run.'); + return; + } } } @@ -50,19 +53,22 @@ function discover_poco_run(&$argv, &$argc){ } else die("Unknown or missing parameter ".$argv[1]."\n"); - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'discover_poco'.$mode.urlencode($search)); - if($pidfile->is_already_running()) { - logger("discover_poco: Already running"); - if ($pidfile->running_time() > 19*60) { - $pidfile->kill(); - logger("discover_poco: killed stale process"); - // Calling a new instance - if ($mode == 0) - proc_run('php','include/discover_poco.php'); + // Don't check this stuff if the function is called by the poller + if (App::callstack() != "poller_run") { + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'discover_poco'.$mode.urlencode($search)); + if($pidfile->is_already_running()) { + logger("discover_poco: Already running"); + if ($pidfile->running_time() > 19*60) { + $pidfile->kill(); + logger("discover_poco: killed stale process"); + // Calling a new instance + if ($mode == 0) + proc_run('php','include/discover_poco.php'); + } + exit; } - exit; } } diff --git a/include/onepoll.php b/include/onepoll.php index 6fb191f73..8b91070dc 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -60,16 +60,19 @@ function onepoll_run(&$argv, &$argc){ return; } - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'onepoll'.$contact_id); - if ($pidfile->is_already_running()) { - logger("onepoll: Already running for contact ".$contact_id); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("killed stale process"); + // Don't check this stuff if the function is called by the poller + if (App::callstack() != "poller_run") { + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'onepoll'.$contact_id); + if ($pidfile->is_already_running()) { + logger("onepoll: Already running for contact ".$contact_id); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("killed stale process"); + } + exit; } - exit; } } diff --git a/include/ostatus.php b/include/ostatus.php index 54b70e6d6..ac13ce6bc 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -954,16 +954,21 @@ function ostatus_completion($conversation_url, $uid, $item = array(), $self = "" if (isset($single_conv->actor->url)) $actor = $single_conv->actor->url; - $contact = q("SELECT `id`, `rel` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'", + $contact = q("SELECT `id`, `rel`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'", $uid, normalise_link($actor), NETWORK_STATUSNET); - if (count($contact)) { + if (!$contact) + $contact = q("SELECT `id`, `rel`, `network` FROM `contact` WHERE `uid` = %d AND `alias` IN ('%s', '%s') AND `network` != '%s'", + $uid, $actor, normalise_link($actor), NETWORK_STATUSNET); + + if ($contact) { logger("Found contact for url ".$actor, LOGGER_DEBUG); $contact_id = $contact[0]["id"]; + $network = $contact[0]["network"]; $not_following = !in_array($contact[0]["rel"], array(CONTACT_IS_SHARING, CONTACT_IS_FRIEND)); } else { - logger("No contact found for url ".$actor, LOGGER_DEBUG); + logger("No contact found for user ".$uid." and url ".$actor, LOGGER_DEBUG); // Adding a global contact /// @TODO Use this data for the post @@ -972,18 +977,19 @@ function ostatus_completion($conversation_url, $uid, $item = array(), $self = "" logger("Global contact ".$global_contact_id." found for url ".$actor, LOGGER_DEBUG); $contact_id = $parent["contact-id"]; + $network = NETWORK_OSTATUS; $not_following = true; } // Do we only want to import threads that were started by our contacts? if ($not_following AND $new_parent AND get_config('system','ostatus_full_threads')) { - logger("Don't import uri ".$first_id." because we don't follow the person ".$actor, LOGGER_DEBUG); + logger("Don't import uri ".$first_id." because user ".$uid." doesn't follow the person ".$actor, LOGGER_DEBUG); continue; } $arr = array(); - $arr["network"] = NETWORK_OSTATUS; + $arr["network"] = $network; $arr["uri"] = $single_conv->id; $arr["plink"] = $plink; $arr["uid"] = $uid; diff --git a/include/pubsubpublish.php b/include/pubsubpublish.php index 0ac50aaaa..e7a55f5f0 100644 --- a/include/pubsubpublish.php +++ b/include/pubsubpublish.php @@ -79,18 +79,21 @@ function pubsubpublish_run(&$argv, &$argc){ load_config('config'); load_config('system'); - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'pubsubpublish'); - if($pidfile->is_already_running()) { - logger("Already running"); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("killed stale process"); - // Calling a new instance - proc_run('php',"include/pubsubpublish.php"); + // Don't check this stuff if the function is called by the poller + if (App::callstack() != "poller_run") { + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'pubsubpublish'); + if($pidfile->is_already_running()) { + logger("Already running"); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("killed stale process"); + // Calling a new instance + proc_run('php',"include/pubsubpublish.php"); + } + return; } - return; } } diff --git a/include/queue.php b/include/queue.php index 1525ca3ab..157fc88d9 100644 --- a/include/queue.php +++ b/include/queue.php @@ -28,18 +28,21 @@ function queue_run(&$argv, &$argc){ load_config('config'); load_config('system'); - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'queue'); - if($pidfile->is_already_running()) { - logger("queue: Already running"); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("queue: killed stale process"); - // Calling a new instance - proc_run('php',"include/queue.php"); + // Don't check this stuff if the function is called by the poller + if (App::callstack() != "poller_run") { + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'queue'); + if($pidfile->is_already_running()) { + logger("queue: Already running"); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("queue: killed stale process"); + // Calling a new instance + proc_run('php',"include/queue.php"); + } + return; } - return; } } diff --git a/include/update_gcontact.php b/include/update_gcontact.php index b5ea30a0a..b7bf25aa2 100644 --- a/include/update_gcontact.php +++ b/include/update_gcontact.php @@ -37,16 +37,19 @@ function update_gcontact_run(&$argv, &$argc){ return; } - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'update_gcontact'.$contact_id); - if ($pidfile->is_already_running()) { - logger("update_gcontact: Already running for contact ".$contact_id); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("killed stale process"); + // Don't check this stuff if the function is called by the poller + if (App::callstack() != "poller_run") { + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'update_gcontact'.$contact_id); + if ($pidfile->is_already_running()) { + logger("update_gcontact: Already running for contact ".$contact_id); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("killed stale process"); + } + exit; } - exit; } } From beb2346cfc8e3aa57ed0203e35034241e814b61a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 8 Mar 2016 20:28:09 +0100 Subject: [PATCH 2/5] The function to check for maxload and the lockfile is centralized --- boot.php | 37 ++++++++++++++++++++++++++++++++++++- include/cron.php | 31 ++++--------------------------- include/cronhooks.php | 31 ++++--------------------------- include/delivery.php | 13 ++----------- include/discover_poco.php | 37 ++++++------------------------------- include/onepoll.php | 18 +++--------------- include/poller.php | 13 ++----------- include/pubsubpublish.php | 20 +++----------------- include/queue.php | 20 +++----------------- include/update_gcontact.php | 18 +++--------------- 10 files changed, 66 insertions(+), 172 deletions(-) diff --git a/boot.php b/boot.php index 5cc349931..fe4e5a275 100644 --- a/boot.php +++ b/boot.php @@ -30,7 +30,7 @@ require_once('include/cache.php'); require_once('library/Mobile_Detect/Mobile_Detect.php'); require_once('include/features.php'); require_once('include/identity.php'); - +require_once('include/pidfile.php'); require_once('update.php'); require_once('include/dbstructure.php'); @@ -1098,6 +1098,41 @@ class App { return($this->is_friendica_app); } + function maxload_reached() { + + $maxsysload = intval(get_config('system', 'maxloadavg')); + if ($maxsysload < 1) + $maxsysload = 50; + + $load = current_load(); + if ($load) { + if (intval($load) > $maxsysload) { + logger('system: load '.$load.' too high.'); + return true; + } + } + return false; + } + + function is_already_running($task, $taskname, $timeout = 540) { + + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, $taskname); + if ($pidfile->is_already_running()) { + logger("Already running"); + if ($pidfile->running_time() > $timeout) { + $pidfile->kill(); + logger("killed stale process"); + // Calling a new instance + if ($task != "") + proc_run('php', $task); + } + return true; + } + } + return false; + } } /** diff --git a/include/cron.php b/include/cron.php index 614328171..1c3297c93 100644 --- a/include/cron.php +++ b/include/cron.php @@ -34,7 +34,6 @@ function cron_run(&$argv, &$argc){ require_once('include/Contact.php'); require_once('include/email.php'); require_once('include/socgraph.php'); - require_once('include/pidfile.php'); require_once('mod/nodeinfo.php'); load_config('config'); @@ -42,32 +41,10 @@ function cron_run(&$argv, &$argc){ // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") { - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; - - $load = current_load(); - if($load) { - if(intval($load) > $maxsysload) { - logger('system: load '.$load.' too high. cron deferred to next scheduled run.'); - return; - } - } - - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'cron'); - if($pidfile->is_already_running()) { - logger("cron: Already running"); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("cron: killed stale process"); - // Calling a new instance - proc_run('php','include/cron.php'); - } - exit; - } - } + if (App::maxload_reached()) + return; + if (App::is_already_running('include/cron.php', 'cron', 540)) + return; } $last = get_config('system','last_cron'); diff --git a/include/cronhooks.php b/include/cronhooks.php index 71cb0fb7b..22812fb86 100644 --- a/include/cronhooks.php +++ b/include/cronhooks.php @@ -19,39 +19,16 @@ function cronhooks_run(&$argv, &$argc){ require_once('include/session.php'); require_once('include/datetime.php'); - require_once('include/pidfile.php'); load_config('config'); load_config('system'); // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") { - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; - - $load = current_load(); - if($load) { - if(intval($load) > $maxsysload) { - logger('system: load ' . $load . ' too high. Cronhooks deferred to next scheduled run.'); - return; - } - } - - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'cronhooks'); - if($pidfile->is_already_running()) { - logger("cronhooks: Already running"); - if ($pidfile->running_time() > 19*60) { - $pidfile->kill(); - logger("cronhooks: killed stale process"); - // Calling a new instance - proc_run('php','include/cronhooks.php'); - } - exit; - } - } + if (App::maxload_reached()) + return; + if (App::is_already_running('include/cronhooks.php', 'cronhooks', 1140)) + return; } $last = get_config('system','last_cronhook'); diff --git a/include/delivery.php b/include/delivery.php index 021ceb996..e5ca0946b 100644 --- a/include/delivery.php +++ b/include/delivery.php @@ -57,17 +57,8 @@ function delivery_run(&$argv, &$argc){ continue; } - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; - - $load = current_load(); - if($load) { - if(intval($load) > $maxsysload) { - logger('system: load ' . $load . ' too high. Delivery deferred to next queue run.'); - return; - } - } + if (App::maxload_reached()) + return; // It's ours to deliver. Remove it from the queue. diff --git a/include/discover_poco.php b/include/discover_poco.php index 550c9897b..8ba2bb236 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -20,25 +20,14 @@ function discover_poco_run(&$argv, &$argc){ require_once('include/session.php'); require_once('include/datetime.php'); - require_once('include/pidfile.php'); load_config('config'); load_config('system'); // Don't check this stuff if the function is called by the poller - if (App::callstack() != "poller_run") { - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; - - $load = current_load(); - if($load) { - if(intval($load) > $maxsysload) { - logger('system: load '.$load.' too high. discover_poco deferred to next scheduled run.'); - return; - } - } - } + if (App::callstack() != "poller_run") + if (App::maxload_reached()) + return; if(($argc > 2) && ($argv[1] == "dirsearch")) { $search = urldecode($argv[2]); @@ -54,23 +43,9 @@ function discover_poco_run(&$argv, &$argc){ die("Unknown or missing parameter ".$argv[1]."\n"); // Don't check this stuff if the function is called by the poller - if (App::callstack() != "poller_run") { - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'discover_poco'.$mode.urlencode($search)); - if($pidfile->is_already_running()) { - logger("discover_poco: Already running"); - if ($pidfile->running_time() > 19*60) { - $pidfile->kill(); - logger("discover_poco: killed stale process"); - // Calling a new instance - if ($mode == 0) - proc_run('php','include/discover_poco.php'); - } - exit; - } - } - } + if (App::callstack() != "poller_run") + if (App::is_already_running('include/discover_poco.php', 'discover_poco'.$mode.urlencode($search), 1140)) + return; $a->set_baseurl(get_config('system','url')); diff --git a/include/onepoll.php b/include/onepoll.php index 8b91070dc..4d270f613 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -31,7 +31,6 @@ function onepoll_run(&$argv, &$argc){ require_once('include/Contact.php'); require_once('include/email.php'); require_once('include/socgraph.php'); - require_once('include/pidfile.php'); require_once('include/queue_fn.php'); load_config('config'); @@ -61,20 +60,9 @@ function onepoll_run(&$argv, &$argc){ } // Don't check this stuff if the function is called by the poller - if (App::callstack() != "poller_run") { - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'onepoll'.$contact_id); - if ($pidfile->is_already_running()) { - logger("onepoll: Already running for contact ".$contact_id); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("killed stale process"); - } - exit; - } - } - } + if (App::callstack() != "poller_run") + if (App::is_already_running('', 'onepoll'.$contact_id, 540)) + return; $d = datetime_convert(); diff --git a/include/poller.php b/include/poller.php index 755862eb6..7ffd47aa6 100644 --- a/include/poller.php +++ b/include/poller.php @@ -29,17 +29,8 @@ function poller_run(&$argv, &$argc){ if (poller_max_connections_reached()) return; - $load = current_load(); - if($load) { - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; - - if(intval($load) > $maxsysload) { - logger('system: load ' . $load . ' too high. poller deferred to next scheduled run.'); - return; - } - } + if (App::maxload_reached()) + return; // Checking the number of workers if (poller_too_much_workers(1)) { diff --git a/include/pubsubpublish.php b/include/pubsubpublish.php index e7a55f5f0..b438b36e6 100644 --- a/include/pubsubpublish.php +++ b/include/pubsubpublish.php @@ -74,28 +74,14 @@ function pubsubpublish_run(&$argv, &$argc){ }; require_once('include/items.php'); - require_once('include/pidfile.php'); load_config('config'); load_config('system'); // Don't check this stuff if the function is called by the poller - if (App::callstack() != "poller_run") { - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'pubsubpublish'); - if($pidfile->is_already_running()) { - logger("Already running"); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("killed stale process"); - // Calling a new instance - proc_run('php',"include/pubsubpublish.php"); - } - return; - } - } - } + if (App::callstack() != "poller_run") + if (App::is_already_running("include/pubsubpublish.php", 'pubsubpublish', 540)) + return; $a->set_baseurl(get_config('system','url')); diff --git a/include/queue.php b/include/queue.php index 157fc88d9..1222199c7 100644 --- a/include/queue.php +++ b/include/queue.php @@ -22,29 +22,15 @@ function queue_run(&$argv, &$argc){ require_once("include/datetime.php"); require_once('include/items.php'); require_once('include/bbcode.php'); - require_once('include/pidfile.php'); require_once('include/socgraph.php'); load_config('config'); load_config('system'); // Don't check this stuff if the function is called by the poller - if (App::callstack() != "poller_run") { - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'queue'); - if($pidfile->is_already_running()) { - logger("queue: Already running"); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("queue: killed stale process"); - // Calling a new instance - proc_run('php',"include/queue.php"); - } - return; - } - } - } + if (App::callstack() != "poller_run") + if (App::is_already_running('include/queue.php', 'queue', 540)) + return; $a->set_baseurl(get_config('system','url')); diff --git a/include/update_gcontact.php b/include/update_gcontact.php index b7bf25aa2..25c11806a 100644 --- a/include/update_gcontact.php +++ b/include/update_gcontact.php @@ -16,7 +16,6 @@ function update_gcontact_run(&$argv, &$argc){ unset($db_host, $db_user, $db_pass, $db_data); }; - require_once('include/pidfile.php'); require_once('include/Scrape.php'); require_once("include/socgraph.php"); @@ -38,20 +37,9 @@ function update_gcontact_run(&$argv, &$argc){ } // Don't check this stuff if the function is called by the poller - if (App::callstack() != "poller_run") { - $lockpath = get_lockpath(); - if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'update_gcontact'.$contact_id); - if ($pidfile->is_already_running()) { - logger("update_gcontact: Already running for contact ".$contact_id); - if ($pidfile->running_time() > 9*60) { - $pidfile->kill(); - logger("killed stale process"); - } - exit; - } - } - } + if (App::callstack() != "poller_run") + if (App::is_already_running('', 'update_gcontact'.$contact_id, 540)) + return; $r = q("SELECT * FROM `gcontact` WHERE `id` = %d", intval($contact_id)); From 65d6d45f8cdb2e1f84fc0c345043598a221312de Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 8 Mar 2016 22:28:49 +0100 Subject: [PATCH 3/5] Changed parameter order --- boot.php | 2 +- include/cron.php | 2 +- include/cronhooks.php | 2 +- include/discover_poco.php | 2 +- include/onepoll.php | 2 +- include/pubsubpublish.php | 2 +- include/queue.php | 2 +- include/update_gcontact.php | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/boot.php b/boot.php index fe4e5a275..4d57d21e6 100644 --- a/boot.php +++ b/boot.php @@ -1114,7 +1114,7 @@ class App { return false; } - function is_already_running($task, $taskname, $timeout = 540) { + function is_already_running($taskname, $task = "", $timeout = 540) { $lockpath = get_lockpath(); if ($lockpath != '') { diff --git a/include/cron.php b/include/cron.php index 1c3297c93..d68bd7f08 100644 --- a/include/cron.php +++ b/include/cron.php @@ -43,7 +43,7 @@ function cron_run(&$argv, &$argc){ if (App::callstack() != "poller_run") { if (App::maxload_reached()) return; - if (App::is_already_running('include/cron.php', 'cron', 540)) + if (App::is_already_running('cron', 'include/cron.php', 540)) return; } diff --git a/include/cronhooks.php b/include/cronhooks.php index 22812fb86..b6cf0e723 100644 --- a/include/cronhooks.php +++ b/include/cronhooks.php @@ -27,7 +27,7 @@ function cronhooks_run(&$argv, &$argc){ if (App::callstack() != "poller_run") { if (App::maxload_reached()) return; - if (App::is_already_running('include/cronhooks.php', 'cronhooks', 1140)) + if (App::is_already_running('cronhooks', 'include/cronhooks.php', 1140)) return; } diff --git a/include/discover_poco.php b/include/discover_poco.php index 8ba2bb236..0b468faea 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -44,7 +44,7 @@ function discover_poco_run(&$argv, &$argc){ // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") - if (App::is_already_running('include/discover_poco.php', 'discover_poco'.$mode.urlencode($search), 1140)) + if (App::is_already_running('discover_poco'.$mode.urlencode($search), 'include/discover_poco.php', 1140)) return; $a->set_baseurl(get_config('system','url')); diff --git a/include/onepoll.php b/include/onepoll.php index 4d270f613..eb1045de1 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -61,7 +61,7 @@ function onepoll_run(&$argv, &$argc){ // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") - if (App::is_already_running('', 'onepoll'.$contact_id, 540)) + if (App::is_already_running('onepoll'.$contact_id, '', 540)) return; $d = datetime_convert(); diff --git a/include/pubsubpublish.php b/include/pubsubpublish.php index b438b36e6..4fbb50514 100644 --- a/include/pubsubpublish.php +++ b/include/pubsubpublish.php @@ -80,7 +80,7 @@ function pubsubpublish_run(&$argv, &$argc){ // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") - if (App::is_already_running("include/pubsubpublish.php", 'pubsubpublish', 540)) + if (App::is_already_running("pubsubpublish", "include/pubsubpublish.php", 540)) return; $a->set_baseurl(get_config('system','url')); diff --git a/include/queue.php b/include/queue.php index 1222199c7..183ce0f9c 100644 --- a/include/queue.php +++ b/include/queue.php @@ -29,7 +29,7 @@ function queue_run(&$argv, &$argc){ // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") - if (App::is_already_running('include/queue.php', 'queue', 540)) + if (App::is_already_running('queue', 'include/queue.php', 540)) return; $a->set_baseurl(get_config('system','url')); diff --git a/include/update_gcontact.php b/include/update_gcontact.php index 25c11806a..88e1817f0 100644 --- a/include/update_gcontact.php +++ b/include/update_gcontact.php @@ -38,7 +38,7 @@ function update_gcontact_run(&$argv, &$argc){ // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") - if (App::is_already_running('', 'update_gcontact'.$contact_id, 540)) + if (App::is_already_running('update_gcontact'.$contact_id, '', 540)) return; $r = q("SELECT * FROM `gcontact` WHERE `id` = %d", intval($contact_id)); From c30b369a1a60af4635b8f0a6ae823303fa50e3da Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 8 Mar 2016 22:42:37 +0100 Subject: [PATCH 4/5] Added documentation --- boot.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/boot.php b/boot.php index 4d57d21e6..43f69fc11 100644 --- a/boot.php +++ b/boot.php @@ -1098,6 +1098,11 @@ class App { return($this->is_friendica_app); } + /** + * @brief Checks if the maximum load is reached + * + * @return bool Is the load reached? + */ function maxload_reached() { $maxsysload = intval(get_config('system', 'maxloadavg')); @@ -1114,6 +1119,15 @@ class App { return false; } + /** + * @brief Checks if the process is already running + * + * @param string $taskname The name of the task that will be used for the name of the lockfile + * @param string $task The path and name of the php script + * @param int $timeout The timeout after which a task should be killed + * + * @return bool Is the process running? + */ function is_already_running($taskname, $task = "", $timeout = 540) { $lockpath = get_lockpath(); From c0bd7a866d7dc0dc0ebc1e82226add912e39cffa Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 9 Mar 2016 21:53:30 +0100 Subject: [PATCH 5/5] Some small addition to the OStatus part. Better check if we follow the actor. --- include/ostatus.php | 86 +++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/include/ostatus.php b/include/ostatus.php index ac13ce6bc..0e79b32f8 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -714,6 +714,49 @@ function ostatus_fetch_conversation($self, $conversation_id = "") { return $base_url."/conversation/".$conversation_id; } +/** + * @brief Fetches actor details of a given actor and user id + * + * @param string $actor The actor url + * @param int $uid The user id + * + * @return array Array with actor details + */ +function ostatus_get_actor_details($actor, $uid) { + + $details = array(); + + $contact = q("SELECT `id`, `rel`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'", + $uid, normalise_link($actor), NETWORK_STATUSNET); + + if (!$contact) + $contact = q("SELECT `id`, `rel`, `network` FROM `contact` WHERE `uid` = %d AND `alias` IN ('%s', '%s') AND `network` != '%s'", + $uid, $actor, normalise_link($actor), NETWORK_STATUSNET); + + if ($contact) { + logger("Found contact for url ".$actor, LOGGER_DEBUG); + $details["contact_id"] = $contact[0]["id"]; + $details["network"] = $contact[0]["network"]; + + $details["not_following"] = !in_array($contact[0]["rel"], array(CONTACT_IS_SHARING, CONTACT_IS_FRIEND)); + } else { + logger("No contact found for user ".$uid." and url ".$actor, LOGGER_DEBUG); + + // Adding a global contact + /// @TODO Use this data for the post + $details["global_contact_id"] = get_contact($actor, 0); + + logger("Global contact ".$global_contact_id." found for url ".$actor, LOGGER_DEBUG); + + $details["contact_id"] = $parent["contact-id"]; + $details["network"] = NETWORK_OSTATUS; + + $details["not_following"] = true; + } + + return $details; +} + function ostatus_completion($conversation_url, $uid, $item = array(), $self = "") { $a = get_app(); @@ -954,46 +997,20 @@ function ostatus_completion($conversation_url, $uid, $item = array(), $self = "" if (isset($single_conv->actor->url)) $actor = $single_conv->actor->url; - $contact = q("SELECT `id`, `rel`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'", - $uid, normalise_link($actor), NETWORK_STATUSNET); - - if (!$contact) - $contact = q("SELECT `id`, `rel`, `network` FROM `contact` WHERE `uid` = %d AND `alias` IN ('%s', '%s') AND `network` != '%s'", - $uid, $actor, normalise_link($actor), NETWORK_STATUSNET); - - if ($contact) { - logger("Found contact for url ".$actor, LOGGER_DEBUG); - $contact_id = $contact[0]["id"]; - $network = $contact[0]["network"]; - - $not_following = !in_array($contact[0]["rel"], array(CONTACT_IS_SHARING, CONTACT_IS_FRIEND)); - } else { - logger("No contact found for user ".$uid." and url ".$actor, LOGGER_DEBUG); - - // Adding a global contact - /// @TODO Use this data for the post - $global_contact_id = get_contact($actor, 0); - - logger("Global contact ".$global_contact_id." found for url ".$actor, LOGGER_DEBUG); - - $contact_id = $parent["contact-id"]; - $network = NETWORK_OSTATUS; - - $not_following = true; - } + $details = ostatus_get_actor_details($actor, $uid); // Do we only want to import threads that were started by our contacts? - if ($not_following AND $new_parent AND get_config('system','ostatus_full_threads')) { + if ($details["not_following"] AND $new_parent AND get_config('system','ostatus_full_threads')) { logger("Don't import uri ".$first_id." because user ".$uid." doesn't follow the person ".$actor, LOGGER_DEBUG); continue; } $arr = array(); - $arr["network"] = $network; + $arr["network"] = $details["network"]; $arr["uri"] = $single_conv->id; $arr["plink"] = $plink; $arr["uid"] = $uid; - $arr["contact-id"] = $contact_id; + $arr["contact-id"] = $details["contact_id"]; $arr["parent-uri"] = $parent_uri; $arr["created"] = $single_conv->published; $arr["edited"] = $single_conv->published; @@ -1119,6 +1136,15 @@ function ostatus_completion($conversation_url, $uid, $item = array(), $self = "" if (($item_stored < 0) AND (count($item) > 0)) { //$arr["app"] .= " (OStatus-NoConvFound)"; + + if (get_config('system','ostatus_full_threads')) { + $details = ostatus_get_actor_details($item["owner-link"], $uid); + if ($details["not_following"]) { + logger("Don't import uri ".$item["uri"]." because user ".$uid." doesn't follow the person ".$item["owner-link"], LOGGER_DEBUG); + return false; + } + } + $item_stored = item_store($item, true); if ($item_stored) { logger("Uri ".$item["uri"]." wasn't found in conversation ".$conversation_url, LOGGER_DEBUG);