From 58cea137075e88a6e86712d3603f8233d7adb56f Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 21 May 2014 07:28:33 +0200 Subject: [PATCH 1/5] Cron/Queue: Setting a limit to avoid endless PHP jobs. --- include/cronhooks.php | 5 ++++- include/queue.php | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/cronhooks.php b/include/cronhooks.php index cf1d13765f..d5daf0d7d4 100644 --- a/include/cronhooks.php +++ b/include/cronhooks.php @@ -50,11 +50,14 @@ function cronhooks_run(&$argv, &$argc){ logger('cronhooks: start'); - $d = datetime_convert(); + set_time_limit(9*60*60); // Setting the maximum execution time for cronjobs to 9 minutes. + call_hooks('cron', $d); + logger('cronhooks: end'); + return; } diff --git a/include/queue.php b/include/queue.php index 4825eb8a4d..7fb6c6d90a 100644 --- a/include/queue.php +++ b/include/queue.php @@ -106,6 +106,8 @@ function queue_run(&$argv, &$argc){ logger('queue: start'); + set_time_limit(9*60*60); // Setting the maximum execution time for queue job to 9 minutes. + handle_pubsubhubbub(); $interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval'))); From e32f9c4fff1f7036c75bc94fcf8bedef700037a8 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 22 May 2014 00:16:36 +0200 Subject: [PATCH 2/5] Dead processes will be deleted when they are running for more than 9 minutes. --- include/cronhooks.php | 10 +++++++--- include/onepoll.php | 2 +- include/pidfile.php | 9 +++++++++ include/poller.php | 8 +++++++- include/queue.php | 10 +++++++--- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/include/cronhooks.php b/include/cronhooks.php index d5daf0d7d4..0ba453555c 100644 --- a/include/cronhooks.php +++ b/include/cronhooks.php @@ -37,9 +37,15 @@ function cronhooks_run(&$argv, &$argc){ $lockpath = get_config('system','lockpath'); if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'cron.lck'); + $pidfile = new pidfile($lockpath, 'cronhooks'); if($pidfile->is_already_running()) { logger("cronhooks: Already running"); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("cronhooks: killed stale process"); + // Calling a new instance + proc_run('php','include/cronhooks.php'); + } exit; } } @@ -52,8 +58,6 @@ function cronhooks_run(&$argv, &$argc){ $d = datetime_convert(); - set_time_limit(9*60*60); // Setting the maximum execution time for cronjobs to 9 minutes. - call_hooks('cron', $d); logger('cronhooks: end'); diff --git a/include/onepoll.php b/include/onepoll.php index 49047118a4..15b8e7c2cc 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -60,7 +60,7 @@ function onepoll_run(&$argv, &$argc){ // Test $lockpath = get_config('system','lockpath'); if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'onepoll'.$contact_id.'.lck'); + $pidfile = new pidfile($lockpath, 'onepoll'.$contact_id); if($pidfile->is_already_running()) { logger("onepoll: Already running for contact ".$contact_id); exit; diff --git a/include/pidfile.php b/include/pidfile.php index 47df8d1f46..4f5b25ad75 100644 --- a/include/pidfile.php +++ b/include/pidfile.php @@ -28,5 +28,14 @@ class pidfile { public function is_already_running() { return $this->_running; } + + public function running_time() { + return(time() - filectime($this->_file)); + } + + public function kill() { + if (file_exists($this->_file)) + return(posix_kill(file_get_contents($this->_file), SIGTERM)); + } } ?> diff --git a/include/poller.php b/include/poller.php index 084067b190..b04c77f744 100644 --- a/include/poller.php +++ b/include/poller.php @@ -43,9 +43,15 @@ function poller_run(&$argv, &$argc){ $lockpath = get_config('system','lockpath'); if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'poller.lck'); + $pidfile = new pidfile($lockpath, 'poller'); if($pidfile->is_already_running()) { logger("poller: Already running"); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("poller: killed stale process"); + // Calling a new instance + proc_run('php','include/poller.php'); + } exit; } } diff --git a/include/queue.php b/include/queue.php index 7fb6c6d90a..bc3cc258cc 100644 --- a/include/queue.php +++ b/include/queue.php @@ -86,9 +86,15 @@ function queue_run(&$argv, &$argc){ $lockpath = get_config('system','lockpath'); if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'queue.lck'); + $pidfile = new pidfile($lockpath, 'queue'); if($pidfile->is_already_running()) { logger("queue: Already running"); + if ($pidfile->running_time() > 9*60) { + $pidfile->kill(); + logger("queue: killed stale process"); + // Calling a new instance + proc_run('php',"include/queue.php"); + } return; } } @@ -106,8 +112,6 @@ function queue_run(&$argv, &$argc){ logger('queue: start'); - set_time_limit(9*60*60); // Setting the maximum execution time for queue job to 9 minutes. - handle_pubsubhubbub(); $interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval'))); From 4924f527d207c05fa4c1fae4f363659bc173b099 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 24 May 2014 11:00:05 +0200 Subject: [PATCH 3/5] plaintext: Moving some code from "plaintext" function to "get_attached_data" to have it accessible for non-plaintext exports --- include/plaintext.php | 68 +++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/include/plaintext.php b/include/plaintext.php index 18763f86aa..d824f87595 100644 --- a/include/plaintext.php +++ b/include/plaintext.php @@ -9,6 +9,10 @@ function get_attached_data($body) { - description: - (thumbnail) */ + + // Simplify image codes + $body = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '[img]$3[/img]', $body); + $post = array(); if (preg_match_all("(\[class=(.*?)\](.*?)\[\/class\])ism",$body, $attached, PREG_SET_ORDER)) { @@ -38,40 +42,11 @@ function get_attached_data($body) { } } - return($post); -} - -function shortenmsg($msg, $limit) { - $lines = explode("\n", $msg); - $msg = ""; - $recycle = html_entity_decode("♲ ", ENT_QUOTES, 'UTF-8'); - foreach ($lines AS $row=>$line) { - if (strlen(trim($msg."\n".$line)) <= $limit) - $msg = trim($msg."\n".$line); - // Is the new message empty by now or is it a reshared message? - elseif (($msg == "") OR (($row == 1) AND (substr($msg, 0, 4) == $recycle))) - $msg = substr(substr(trim($msg."\n".$line), 0, $limit), 0, -3)."..."; - else - break; - } - return($msg); -} - -function plaintext($a, $b, $limit = 0, $includedlinks = false) { - require_once("include/bbcode.php"); - require_once("include/html2plain.php"); - require_once("mod/parse_url.php"); - require_once("include/network.php"); - - // Simplify image codes - $body = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '[img]$3[/img]', $b["body"]); - - // At first look at data that is attached via "type-..." stuff - // This will hopefully replaced with a dedicated bbcode later - $post = get_attached_data($body); // if nothing is found, it maybe having an image. if (!isset($post["type"])) { + require_once("mod/parse_url.php"); + $URLSearchString = "^\[\]"; if (preg_match_all("(\[url=([$URLSearchString]*)\]\s*\[img\]([$URLSearchString]*)\[\/img\]\s*\[\/url\])ism", $body, $pictures, PREG_SET_ORDER)) { if (count($pictures) == 1) { @@ -125,6 +100,37 @@ function plaintext($a, $b, $limit = 0, $includedlinks = false) { } } + return($post); +} + +function shortenmsg($msg, $limit, $twitter = false) { + // To-Do: + // For Twitter URLs aren't shortened, but they have to be calculated as if. + + $lines = explode("\n", $msg); + $msg = ""; + $recycle = html_entity_decode("♲ ", ENT_QUOTES, 'UTF-8'); + foreach ($lines AS $row=>$line) { + if (strlen(trim($msg."\n".$line)) <= $limit) + $msg = trim($msg."\n".$line); + // Is the new message empty by now or is it a reshared message? + elseif (($msg == "") OR (($row == 1) AND (substr($msg, 0, 4) == $recycle))) + $msg = substr(substr(trim($msg."\n".$line), 0, $limit), 0, -3)."..."; + else + break; + } + return($msg); +} + +function plaintext($a, $b, $limit = 0, $includedlinks = false) { + require_once("include/bbcode.php"); + require_once("include/html2plain.php"); + require_once("include/network.php"); + + // At first look at data that is attached via "type-..." stuff + // This will hopefully replaced with a dedicated bbcode later + $post = get_attached_data($b["body"]); + if (($b["title"] != "") AND ($post["text"] != "")) $post["text"] = trim($b["title"]."\n\n".$post["text"]); elseif ($b["title"] != "") From 5ae9c5e538391dbf4e045763d318215560bc6a76 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 24 May 2014 11:01:33 +0200 Subject: [PATCH 4/5] Added some logging --- include/network.php | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/include/network.php b/include/network.php index 240d2978f1..fe08fc5a7b 100644 --- a/include/network.php +++ b/include/network.php @@ -122,7 +122,6 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_ if(! function_exists('post_url')) { function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) { - $stamp1 = microtime(true); $a = get_app(); @@ -130,6 +129,8 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) if(($redirects > 8) || (! $ch)) return false; + logger("post_url: start ".$url, LOGGER_DATA); + curl_setopt($ch, CURLOPT_HEADER, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER,true); curl_setopt($ch, CURLOPT_POST,1); @@ -178,6 +179,8 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) $curl_info = curl_getinfo($ch); $http_code = $curl_info['http_code']; + logger("post_url: result ".$http_code." - ".$url, LOGGER_DATA); + $header = ''; // Pull out multiple headers, e.g. proxy and continuation headers @@ -190,16 +193,18 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) } if($http_code == 301 || $http_code == 302 || $http_code == 303 || $http_code == 307) { - $matches = array(); - preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches); - $newurl = trim(array_pop($matches)); + $matches = array(); + preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches); + $newurl = trim(array_pop($matches)); if(strpos($newurl,'/') === 0) $newurl = $old_location_info["scheme"] . "://" . $old_location_info["host"] . $newurl; - if (filter_var($newurl, FILTER_VALIDATE_URL)) { - $redirects++; - return fetch_url($newurl,false,$redirects,$timeout); - } - } + if (filter_var($newurl, FILTER_VALIDATE_URL)) { + $redirects++; + logger("post_url: redirect ".$url." to ".$newurl); + return post_url($newurl,$params, $headers, $redirects, $timeout); + //return fetch_url($newurl,false,$redirects,$timeout); + } + } $a->set_curl_code($http_code); $body = substr($s,strlen($header)); @@ -209,6 +214,8 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) $a->save_timestamp($stamp1, "network"); + logger("post_url: end ".$url, LOGGER_DATA); + return($body); }} From 235eae66bb659152aa716b487c07d26721b5e6b8 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 24 May 2014 11:01:52 +0200 Subject: [PATCH 5/5] New query parameter that Speeds up the network viee when there are many contacts --- mod/network.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/network.php b/mod/network.php index 585c063f56..ab44073508 100644 --- a/mod/network.php +++ b/mod/network.php @@ -710,7 +710,7 @@ die("ss"); ); } else { $r = q("SELECT `thread`.`iid` AS `item_id`, `thread`.`network` AS `item_network`, `contact`.`uid` AS `contact_uid` - FROM $sql_table $sql_post_table INNER JOIN `contact` ON `contact`.`id` = `thread`.`contact-id` + FROM $sql_table $sql_post_table STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id` AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0 WHERE `thread`.`uid` = %d AND `thread`.`visible` = 1 AND `thread`.`deleted` = 0 AND `thread`.`moderated` = 0