From 44efdc3e1b01328a226c79e25834fe7adb2dbaf4 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 20 May 2017 20:02:06 +0000 Subject: [PATCH 01/25] Central item expiration routine for external items --- doc/htconfig.md | 1 + include/dbclean.php | 79 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/doc/htconfig.md b/doc/htconfig.md index b2f718296..ce0b446de 100644 --- a/doc/htconfig.md +++ b/doc/htconfig.md @@ -35,6 +35,7 @@ Example: To set the directory value please add this line to your .htconfig.php: * **db_loglimit_index_high** - Number of index rows to be logged anyway (for any index) * **db_log_index_blacklist** - Blacklist of indexes that shouldn't be watched * **dbclean** (Boolean) - Enable the automatic database cleanup process +* **dbclean-expire-days** (Integer) - Days after which remote items will be deleted. Own items, and marked or filed items are kept. * **default_service_class** - * **delivery_batch_count** - Number of deliveries per process. Default value is 1. (Disabled when using the worker) * **diaspora_test** (Boolean) - For development only. Disables the message transfer. diff --git a/include/dbclean.php b/include/dbclean.php index ff4993c4a..024b845f7 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -17,9 +17,14 @@ function dbclean_run(&$argv, &$argc) { $stage = 0; } + // Get the expire days for step 8 and 9 + $days = Config::get('system', 'dbclean-expire-days', 0); + if ($stage == 0) { - for ($i = 1; $i <= 7; $i++) { - if (!Config::get('system', 'finished-dbclean-'.$i, false)) { + for ($i = 1; $i <= 9; $i++) { + // Execute the background script for a step when it isn't finished. + // Execute step 8 and 9 only when $days is defined. + if (!Config::get('system', 'finished-dbclean-'.$i, false) AND (($i < 8) OR ($days > 0))) { proc_run(PRIORITY_LOW, 'include/dbclean.php', $i); } } @@ -39,6 +44,9 @@ function remove_orphans($stage = 0) { // We split the deletion in many small tasks $limit = 1000; + // Get the expire days for step 8 and 9 + $days = Config::get('system', 'dbclean-expire-days', 0); + if ($stage == 1) { $last_id = Config::get('system', 'dbclean-last-id-1', 0); @@ -61,11 +69,6 @@ function remove_orphans($stage = 0) { logger("Done deleting ".$count." old global item entries from item table without user copy. Last ID: ".$last_id); Config::set('system', 'dbclean-last-id-1', $last_id); - - // We will eventually set this value when we found a good way to delete these items in another way. - // if ($count < $limit) { - // Config::set('system', 'finished-dbclean-1', true); - // } } elseif ($stage == 2) { $last_id = Config::get('system', 'dbclean-last-id-2', 0); @@ -216,11 +219,71 @@ function remove_orphans($stage = 0) { if ($count < $limit) { Config::set('system', 'finished-dbclean-7', true); } + } elseif ($stage == 8) { + if ($days <= 0) { + return; + } + + $last_id = Config::get('system', 'dbclean-last-id-8', 0); + + logger("Deleting expired threads. Last ID: ".$last_id); + $r = dba::p("SELECT `thread`.`iid` FROM `thread` + INNER JOIN `contact` ON `thread`.`contact-id` = `contact`.`id` AND NOT `notify_new_posts` + WHERE `thread`.`received` < UTC_TIMESTAMP() - INTERVAL ? DAY + AND NOT `thread`.`mention` AND NOT `thread`.`starred` + AND NOT `thread`.`wall` AND NOT `thread`.`origin` + AND `thread`.`uid` != 0 AND `thread`.`iid` >= ? + AND NOT `thread`.`iid` IN (SELECT `parent` FROM `item` + WHERE (`item`.`starred` OR (`item`.`resource-id` != '') + OR (`item`.`file` != '') OR (`item`.`event-id` != '') + OR (`item`.`attach` != '') OR `item`.`wall` OR `item`.`origin`) + AND `item`.`parent` = `thread`.`iid`) + ORDER BY `thread`.`iid` LIMIT 1000", $days, $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found expired threads: ".$count); + while ($thread = dba::fetch($r)) { + $last_id = $thread["iid"]; + dba::delete('thread', array('iid' => $thread["iid"])); + } + } else { + logger("No expired threads found"); + } + dba::close($r); + logger("Done deleting ".$count." expired threads. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-8', $last_id); + } elseif ($stage == 9) { + if ($days <= 0) { + return; + } + + $last_id = Config::get('system', 'dbclean-last-id-9', 0); + $till_id = Config::get('system', 'dbclean-last-id-8', 0); + + logger("Deleting old global item entries from expired threads from ID ".$last_id." to ID ".$till_id); + $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 AND + NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) AND + `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY AND `id` >= ? AND `id` <= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id, $till_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found global item entries from expired threads: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; + dba::delete('item', array('id' => $orphan["id"])); + } + } else { + logger("No global item entries from expired threads"); + } + dba::close($r); + logger("Done deleting ".$count." old global item entries from expired threads. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-9', $last_id); } // Call it again if not all entries were purged if (($stage != 0) AND ($count > 0)) { proc_run(PRIORITY_MEDIUM, 'include/dbclean.php'); } - } From 32f61016d368401890443efc861fff0032ac6c42 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Jun 2017 07:26:21 +0000 Subject: [PATCH 02/25] Overhauled "lock" functionality --- include/dbstructure.php | 2 +- include/lock.php | 80 ----------------------------------------- src/Util/Lock.php | 62 ++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 81 deletions(-) delete mode 100644 include/lock.php create mode 100644 src/Util/Lock.php diff --git a/include/dbstructure.php b/include/dbstructure.php index 441e7be7f..1801288a4 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -1205,7 +1205,7 @@ function db_definition() { "id" => array("type" => "int(11)", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), "name" => array("type" => "varchar(128)", "not null" => "1", "default" => ""), "locked" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), - "created" => array("type" => "datetime", "default" => NULL_DATE), + "pid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"), ), "indexes" => array( "PRIMARY" => array("id"), diff --git a/include/lock.php b/include/lock.php deleted file mode 100644 index 64f6319ef..000000000 --- a/include/lock.php +++ /dev/null @@ -1,80 +0,0 @@ - $fn_name), array('limit' => 1)); + + if ((dbm::is_result($lock)) AND !$lock['locked']) { + dba::update('locks', array('locked' => true), array('name' => $fn_name)); + $got_lock = true; + } elseif (!dbm::is_result($lock)) { + dbm::insert('locks', array('name' => $fn_name, 'locked' => true)); + $got_lock = true; + } + + dbm::p("UNLOCK TABLES"); + + if (!$got_lock) { + sleep($wait_sec); + } + } while (!$got_lock AND ((time() - $start) < $timeout)); + + logger('lock_function: function ' . $fn_name . ' with blocking = ' . $block . ' got_lock = ' . $got_lock . ' time = ' . (time() - $start), LOGGER_DEBUG); + + return $got_lock; + } + + public static function remove($fn_name) { + dba::update('locks', array('locked' => false), array('name' => $fn_name)); + + logger('unlock_function: released lock for function ' . $fn_name, LOGGER_DEBUG); + + return; + } +} From 5de03c1b271e492b861e22c7f510fbb6ebec82a6 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Jun 2017 08:36:08 +0000 Subject: [PATCH 03/25] Locks now work - ready to be used in the wild --- src/Util/Lock.php | 53 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/src/Util/Lock.php b/src/Util/Lock.php index 03e8545ad..e8fa03f78 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -16,11 +16,16 @@ use dbm; */ class Lock { - -// Provide some ability to lock a PHP function so that multiple processes -// can't run the function concurrently - - public static function set($fn_name, $wait_sec = 2, $timeout = 30) { + /** + * @brief Sets a lock for a given name + * + * @param string $fn_name Name of the lock + * @param integer $timeout Seconds until we give up + * @param integer $wait_sec Time between to lock attempts + * + * @return boolean Was the lock successful? + */ + public static function set($fn_name, $timeout = 30, $wait_sec = 2) { if ($wait_sec == 0) { $wait_sec = 2; } @@ -29,34 +34,46 @@ class Lock { $start = time(); do { - dba:p("LOCK TABLE `locks` WRITE"); - $lock = dba::select('locks', array('locked'), array('name' => $fn_name), array('limit' => 1)); + dba::p("LOCK TABLE `locks` WRITE"); + $lock = dba::select('locks', array('locked', 'pid'), array('name' => $fn_name), array('limit' => 1)); - if ((dbm::is_result($lock)) AND !$lock['locked']) { - dba::update('locks', array('locked' => true), array('name' => $fn_name)); - $got_lock = true; + if (dbm::is_result($lock)) { + if ($lock['locked']) { + // When the process id isn't used anymore, we can safely claim the lock for us. + if (!posix_kill($lock['pid'], 0)) { + $lock['locked'] = false; + } + // We want to lock something that was already locked by us? So we got the lock. + if ($lock['pid'] == getmypid()) { + $got_lock = true; + } + } + if (!$lock['locked']) { + dba::update('locks', array('locked' => true, 'pid' => getmypid()), array('name' => $fn_name)); + $got_lock = true; + } } elseif (!dbm::is_result($lock)) { - dbm::insert('locks', array('name' => $fn_name, 'locked' => true)); + dba::insert('locks', array('name' => $fn_name, 'locked' => true, 'pid' => getmypid())); $got_lock = true; } - dbm::p("UNLOCK TABLES"); + dba::p("UNLOCK TABLES"); if (!$got_lock) { sleep($wait_sec); } } while (!$got_lock AND ((time() - $start) < $timeout)); - logger('lock_function: function ' . $fn_name . ' with blocking = ' . $block . ' got_lock = ' . $got_lock . ' time = ' . (time() - $start), LOGGER_DEBUG); - return $got_lock; } + /** + * @brief Removes a lock if it was set by us + * + * @param string $fn_name Name of the lock + */ public static function remove($fn_name) { - dba::update('locks', array('locked' => false), array('name' => $fn_name)); - - logger('unlock_function: released lock for function ' . $fn_name, LOGGER_DEBUG); - + dba::update('locks', array('locked' => false, 'pid' => 0), array('name' => $fn_name, 'pid' => getmypid())); return; } } From d2cb87a200cadcf055b194ea0b2fc577640fdea2 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Jun 2017 12:59:29 +0000 Subject: [PATCH 04/25] Database locks are now having its very own functions --- include/dba.php | 22 ++++++++++++++++++++++ src/Util/Lock.php | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/include/dba.php b/include/dba.php index b9e6c32d5..1fff51f3d 100644 --- a/include/dba.php +++ b/include/dba.php @@ -806,6 +806,28 @@ class dba { return self::e($sql, $param); } + /** + * @brief Locks a table for exclusive write access + * + * This function can be extended in the future to accept a table array as well. + * + * @param string $table Table name + * + * @return boolean was the lock successful? + */ + static public function lock($table) { + return self::e("LOCK TABLES `".self::$dbo->escape($table)."` WRITE"); + } + + /** + * @brief Unlocks all locked tables + * + * @return boolean was the unlock successful? + */ + static public function unlock() { + return self::e("UNLOCK TABLES"); + } + /** * @brief Starts a transaction * diff --git a/src/Util/Lock.php b/src/Util/Lock.php index e8fa03f78..1a33e819f 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -34,7 +34,7 @@ class Lock { $start = time(); do { - dba::p("LOCK TABLE `locks` WRITE"); + dba::lock('locks'); $lock = dba::select('locks', array('locked', 'pid'), array('name' => $fn_name), array('limit' => 1)); if (dbm::is_result($lock)) { @@ -57,7 +57,7 @@ class Lock { $got_lock = true; } - dba::p("UNLOCK TABLES"); + dba::unlock(); if (!$got_lock) { sleep($wait_sec); From bca5776e9c7f84d95fd7066ee2948f9ef945f78f Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Jun 2017 15:59:20 +0000 Subject: [PATCH 05/25] Lock now can use the memcache as well --- include/poller.php | 35 +++++++++++++------- src/Util/Lock.php | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 12 deletions(-) diff --git a/include/poller.php b/include/poller.php index 22fb65a4f..60937fb7e 100644 --- a/include/poller.php +++ b/include/poller.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\Config; +use Friendica\Util\Lock; if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) { $directory = dirname($_SERVER["argv"][0]); @@ -19,16 +20,12 @@ require_once("boot.php"); function poller_run($argv, $argc){ global $a, $db; - if (is_null($a)) { - $a = new App(dirname(__DIR__)); - } + $a = new App(dirname(__DIR__)); - if(is_null($db)) { - @include(".htconfig.php"); - require_once("include/dba.php"); - $db = new dba($db_host, $db_user, $db_pass, $db_data); - unset($db_host, $db_user, $db_pass, $db_data); - }; + @include(".htconfig.php"); + require_once("include/dba.php"); + $db = new dba($db_host, $db_user, $db_pass, $db_data); + unset($db_host, $db_user, $db_pass, $db_data); Config::load(); @@ -41,6 +38,10 @@ function poller_run($argv, $argc){ load_hooks(); + if (($argc <= 1) OR ($argv[1] != "no_cron")) { + poller_run_cron(); + } + $a->start_process(); if ($a->min_memory_reached()) { @@ -55,11 +56,11 @@ function poller_run($argv, $argc){ return; } - if(($argc <= 1) OR ($argv[1] != "no_cron")) { - poller_run_cron(); + if ($a->max_processes_reached()) { + return; } - if ($a->max_processes_reached()) { + if (!Lock::set('poller_worker')) { return; } @@ -69,6 +70,8 @@ function poller_run($argv, $argc){ return; } + Lock::remove('poller_worker'); + $starttime = time(); while ($r = poller_worker_process()) { @@ -83,12 +86,18 @@ function poller_run($argv, $argc){ return; } + if (!Lock::set('poller_worker')) { + return; + } + // Count active workers and compare them with a maximum value that depends on the load if (poller_too_much_workers()) { logger('Active worker limit reached, quitting.', LOGGER_DEBUG); return; } + Lock::remove('poller_worker'); + if (!poller_execute($r[0])) { logger('Process execution failed, quitting.', LOGGER_DEBUG); return; @@ -724,6 +733,8 @@ function poller_run_cron() { if (array_search(__file__,get_included_files())===0){ poller_run($_SERVER["argv"],$_SERVER["argc"]); + Lock::removeAll(); + poller_unclaim_process(); get_app()->end_process(); diff --git a/src/Util/Lock.php b/src/Util/Lock.php index 1a33e819f..175ad34e3 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -8,6 +8,8 @@ namespace Friendica\Util; * */ +use Friendica\Core\Config; +use Memcache; use dba; use dbm; @@ -15,6 +17,31 @@ use dbm; * @brief This class contain Functions for preventing parallel execution of functions */ class Lock { + /** + * @brief Check for memcache and open a connection if configured + * + * @return object|boolean The memcache object - or "false" if not successful + */ + public static function memcache() { + if (!function_exists('memcache_connect')) { + return false; + } + + if (!Config::get('system', 'memcache')) { + return false; + } + + $memcache_host = Config::get('system', 'memcache_host', '127.0.0.1'); + $memcache_port = Config::get('system', 'memcache_port', 11211); + + $memcache = new Memcache; + + if (!$memcache->connect($memcache_host, $memcache_port)) { + return false; + } + + return $memcache; + } /** * @brief Sets a lock for a given name @@ -33,6 +60,33 @@ class Lock { $got_lock = false; $start = time(); + $memcache = self::memcache(); + if (is_object($memcache)) { + $cachekey = get_app()->get_hostname().";lock:".$fn_name; + + do { + $lock = $memcache->get($cachekey); + + if (!is_bool($lock)) { + $pid = (int)$lock; + + // When the process id isn't used anymore, we can safely claim the lock for us. + // Or we do want to lock something that was already locked by us. + if (!posix_kill($pid, 0) OR ($pid == getmypid())) { + $lock = false; + } + } + if (is_bool($lock)) { + $memcache->set($cachekey, getmypid(), MEMCACHE_COMPRESSED, 300); + $got_lock = true; + } + if (!$got_lock) { + sleep($wait_sec); + } + } while (!$got_lock AND ((time() - $start) < $timeout)); + + return $got_lock; + } do { dba::lock('locks'); $lock = dba::select('locks', array('locked', 'pid'), array('name' => $fn_name), array('limit' => 1)); @@ -73,7 +127,34 @@ class Lock { * @param string $fn_name Name of the lock */ public static function remove($fn_name) { + $memcache = self::memcache(); + if (is_object($memcache)) { + $cachekey = get_app()->get_hostname().";lock:".$fn_name; + $lock = $memcache->get($cachekey); + + if (!is_bool($lock)) { + if ((int)$lock == getmypid()) { + $memcache->delete($cachekey); + } + } + return; + } + dba::update('locks', array('locked' => false, 'pid' => 0), array('name' => $fn_name, 'pid' => getmypid())); return; } + + /** + * @brief Removes all lock that were set by us + */ + public static function removeAll() { + $memcache = self::memcache(); + if (is_object($memcache)) { + // We cannot delete all cache entries, but this doesn't matter with memcache + return; + } + + dba::update('locks', array('locked' => false, 'pid' => 0), array('pid' => getmypid())); + return; + } } From 30b24a2908513d64ba2a81f1120bca7c08c46b65 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Jun 2017 18:59:50 +0000 Subject: [PATCH 06/25] Locking seems to work great now --- include/poller.php | 19 ++++++++++--------- src/Util/Lock.php | 11 +++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/poller.php b/include/poller.php index 60937fb7e..0d0760588 100644 --- a/include/poller.php +++ b/include/poller.php @@ -45,33 +45,32 @@ function poller_run($argv, $argc){ $a->start_process(); if ($a->min_memory_reached()) { + logger('Pre check: Memory limit reached, quitting.', LOGGER_DEBUG); return; } if (poller_max_connections_reached()) { + logger('Pre check: maximum connections reached, quitting.', LOGGER_DEBUG); return; } if ($a->maxload_reached()) { + logger('Pre check: maximum load reached, quitting.', LOGGER_DEBUG); return; } if ($a->max_processes_reached()) { - return; - } - - if (!Lock::set('poller_worker')) { + logger('Pre check: maximum processes reached, quitting.', LOGGER_DEBUG); return; } // Checking the number of workers if (poller_too_much_workers()) { poller_kill_stale_workers(); + logger('Pre check: Active worker limit reached, quitting.', LOGGER_DEBUG); return; } - Lock::remove('poller_worker'); - $starttime = time(); while ($r = poller_worker_process()) { @@ -87,7 +86,9 @@ function poller_run($argv, $argc){ } if (!Lock::set('poller_worker')) { - return; + logger('Cannot get a lock, retrying.', LOGGER_DEBUG); + poller_unclaim_process(); + continue; } // Count active workers and compare them with a maximum value that depends on the load @@ -104,7 +105,7 @@ function poller_run($argv, $argc){ } // Quit the poller once every hour - if (time() > ($starttime + 3600)) { + if (time() > ($starttime + 360)) { logger('Process lifetime reachted, quitting.', LOGGER_DEBUG); return; } @@ -733,7 +734,7 @@ function poller_run_cron() { if (array_search(__file__,get_included_files())===0){ poller_run($_SERVER["argv"],$_SERVER["argc"]); - Lock::removeAll(); + Lock::remove('poller_worker'); poller_unclaim_process(); diff --git a/src/Util/Lock.php b/src/Util/Lock.php index 175ad34e3..e8011bf59 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -48,20 +48,16 @@ class Lock { * * @param string $fn_name Name of the lock * @param integer $timeout Seconds until we give up - * @param integer $wait_sec Time between to lock attempts * * @return boolean Was the lock successful? */ - public static function set($fn_name, $timeout = 30, $wait_sec = 2) { - if ($wait_sec == 0) { - $wait_sec = 2; - } - + public static function set($fn_name, $timeout = 120) { $got_lock = false; $start = time(); $memcache = self::memcache(); if (is_object($memcache)) { + $wait_sec = 1; $cachekey = get_app()->get_hostname().";lock:".$fn_name; do { @@ -87,6 +83,9 @@ class Lock { return $got_lock; } + + $wait_sec = 2; + do { dba::lock('locks'); $lock = dba::select('locks', array('locked', 'pid'), array('name' => $fn_name), array('limit' => 1)); From bde4943da51aac866ed0870617b868d779bc6835 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Jun 2017 19:01:22 +0000 Subject: [PATCH 07/25] Every hour should mean: every hour :) --- include/poller.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/poller.php b/include/poller.php index 0d0760588..bae2ac5e9 100644 --- a/include/poller.php +++ b/include/poller.php @@ -105,7 +105,7 @@ function poller_run($argv, $argc){ } // Quit the poller once every hour - if (time() > ($starttime + 360)) { + if (time() > ($starttime + 3600)) { logger('Process lifetime reachted, quitting.', LOGGER_DEBUG); return; } From 16276b21ebf617c5c2277b0c25f43f74131d5778 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Jun 2017 19:05:15 +0000 Subject: [PATCH 08/25] Typo --- include/poller.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/poller.php b/include/poller.php index bae2ac5e9..087b4b733 100644 --- a/include/poller.php +++ b/include/poller.php @@ -106,7 +106,7 @@ function poller_run($argv, $argc){ // Quit the poller once every hour if (time() > ($starttime + 3600)) { - logger('Process lifetime reachted, quitting.', LOGGER_DEBUG); + logger('Process lifetime reached, quitting.', LOGGER_DEBUG); return; } } From fb72fc77f5268a9036b43eabea2fa5fffff57b68 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Jun 2017 20:03:37 +0000 Subject: [PATCH 09/25] Some code beautification --- include/poller.php | 160 +++++++++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 71 deletions(-) diff --git a/include/poller.php b/include/poller.php index a67f6a7d8..3d53be0ab 100644 --- a/include/poller.php +++ b/include/poller.php @@ -7,9 +7,9 @@ use Friendica\Util\Lock; if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) { $directory = dirname($_SERVER["argv"][0]); - if (substr($directory, 0, 1) != "/") + if (substr($directory, 0, 1) != "/") { $directory = $_SERVER["PWD"]."/".$directory; - + } $directory = realpath($directory."/.."); chdir($directory); @@ -38,53 +38,58 @@ function poller_run($argv, $argc){ load_hooks(); - if (($argc <= 1) OR ($argv[1] != "no_cron")) { - poller_run_cron(); - } - - $a->start_process(); - - if ($a->min_memory_reached()) { - logger('Pre check: Memory limit reached, quitting.', LOGGER_DEBUG); - return; - } - - if (poller_max_connections_reached()) { - logger('Pre check: maximum connections reached, quitting.', LOGGER_DEBUG); - return; - } - + // At first check the maximum load. We shouldn't continue with a high load if ($a->maxload_reached()) { logger('Pre check: maximum load reached, quitting.', LOGGER_DEBUG); return; } - if ($a->max_processes_reached()) { - logger('Pre check: maximum processes reached, quitting.', LOGGER_DEBUG); - return; - } + // We now start the process. This is done after the load check since this could increase the load. + $a->start_process(); - // Checking the number of workers + // At first we check the number of workers and quit if there are too much of them + // This is done at the top to avoid that too much code is executed without a need to do so, + // since the poller mostly quits here. if (poller_too_much_workers()) { poller_kill_stale_workers(); logger('Pre check: Active worker limit reached, quitting.', LOGGER_DEBUG); return; } + // Do we have too few memory? + if ($a->min_memory_reached()) { + logger('Pre check: Memory limit reached, quitting.', LOGGER_DEBUG); + return; + } + + // Possibly there are too much database connections + if (poller_max_connections_reached()) { + logger('Pre check: maximum connections reached, quitting.', LOGGER_DEBUG); + return; + } + + // Possibly there are too much database processes that block the system + if ($a->max_processes_reached()) { + logger('Pre check: maximum processes reached, quitting.', LOGGER_DEBUG); + return; + } + + // Now we start additional cron processes if we should do so + if (($argc <= 1) OR ($argv[1] != "no_cron")) { + poller_run_cron(); + } + $starttime = time(); + // We fetch the next queue entry that is about to be executed while ($r = poller_worker_process()) { + // If we got that queue entry we claim it for us if (!poller_claim_process($r[0])) { continue; } - // Check free memory - if ($a->min_memory_reached()) { - logger('Memory limit reached, quitting.', LOGGER_DEBUG); - return; - } - + // To avoid the quitting of multiple pollers we serialize the next check if (!Lock::set('poller_worker')) { logger('Cannot get a lock, retrying.', LOGGER_DEBUG); poller_unclaim_process(); @@ -99,6 +104,13 @@ function poller_run($argv, $argc){ Lock::remove('poller_worker'); + // Check free memory + if ($a->min_memory_reached()) { + logger('Memory limit reached, quitting.', LOGGER_DEBUG); + return; + } + + // finally the work will be done if (!poller_execute($r[0])) { logger('Process execution failed, quitting.', LOGGER_DEBUG); return; @@ -160,7 +172,6 @@ function poller_execute($queue) { $funcname = str_replace(".php", "", basename($argv[0]))."_run"; if (function_exists($funcname)) { - poller_exec_function($queue, $funcname, $argv); dba::delete('workerqueue', array('id' => $queue["id"])); } else { @@ -236,24 +247,27 @@ function poller_exec_function($queue, $funcname, $argv) { $o = "\nDatabase Read:\n"; foreach ($a->callstack["database"] AS $func => $time) { $time = round($time, 3); - if ($time > 0) + if ($time > 0) { $o .= $func.": ".$time."\n"; + } } } if (isset($a->callstack["database_write"])) { $o .= "\nDatabase Write:\n"; foreach ($a->callstack["database_write"] AS $func => $time) { $time = round($time, 3); - if ($time > 0) + if ($time > 0) { $o .= $func.": ".$time."\n"; + } } } if (isset($a->callstack["network"])) { $o .= "\nNetwork:\n"; foreach ($a->callstack["network"] AS $func => $time) { $time = round($time, 3); - if ($time > 0) + if ($time > 0) { $o .= $func.": ".$time."\n"; + } } } } else { @@ -294,27 +308,30 @@ function poller_max_connections_reached() { if ($max == 0) { // the maximum number of possible user connections can be a system variable $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_user_connections'"); - if ($r) + if (dbm::is_result($r)) { $max = $r[0]["Value"]; - + } // Or it can be granted. This overrides the system variable $r = q("SHOW GRANTS"); - if ($r) + if (dbm::is_result($r)) { foreach ($r AS $grants) { $grant = array_pop($grants); - if (stristr($grant, "GRANT USAGE ON")) - if (preg_match("/WITH MAX_USER_CONNECTIONS (\d*)/", $grant, $match)) + if (stristr($grant, "GRANT USAGE ON")) { + if (preg_match("/WITH MAX_USER_CONNECTIONS (\d*)/", $grant, $match)) { $max = $match[1]; + } + } } + } } // If $max is set we will use the processlist to determine the current number of connections // The processlist only shows entries of the current user if ($max != 0) { $r = q("SHOW PROCESSLIST"); - if (!dbm::is_result($r)) + if (!dbm::is_result($r)) { return false; - + } $used = count($r); logger("Connection usage (user values): ".$used."/".$max, LOGGER_DEBUG); @@ -330,28 +347,28 @@ function poller_max_connections_reached() { // We will now check for the system values. // This limit could be reached although the user limits are fine. $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_connections'"); - if (!$r) + if (!dbm::is_result($r)) { return false; - + } $max = intval($r[0]["Value"]); - if ($max == 0) + if ($max == 0) { return false; - + } $r = q("SHOW STATUS WHERE `variable_name` = 'Threads_connected'"); - if (!$r) + if (!dbm::is_result($r)) { return false; - + } $used = intval($r[0]["Value"]); - if ($used == 0) + if ($used == 0) { return false; - + } logger("Connection usage (system values): ".$used."/".$max, LOGGER_DEBUG); $level = $used / $max * 100; - if ($level < $maxlevel) + if ($level < $maxlevel) { return false; - + } logger("Maximum level (".$level."%) of system connections reached: ".$used."/".$max); return true; } @@ -376,9 +393,9 @@ function poller_kill_stale_workers() { // Kill long running processes // Check if the priority is in a valid range - if (!in_array($pid["priority"], array(PRIORITY_CRITICAL, PRIORITY_HIGH, PRIORITY_MEDIUM, PRIORITY_LOW, PRIORITY_NEGLIGIBLE))) + if (!in_array($pid["priority"], array(PRIORITY_CRITICAL, PRIORITY_HIGH, PRIORITY_MEDIUM, PRIORITY_LOW, PRIORITY_NEGLIGIBLE))) { $pid["priority"] = PRIORITY_MEDIUM; - + } // Define the maximum durations $max_duration_defaults = array(PRIORITY_CRITICAL => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 360); $max_duration = $max_duration_defaults[$pid["priority"]]; @@ -480,7 +497,7 @@ function poller_too_much_workers() { } } - return($active >= $queues); + return $active >= $queues; } /** @@ -491,7 +508,7 @@ function poller_too_much_workers() { function poller_active_workers() { $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'poller.php'"); - return($workers[0]["processes"]); + return $workers[0]["processes"]; } /** @@ -512,36 +529,37 @@ function poller_passing_slow(&$highest_priority) { INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid`"); // No active processes at all? Fine - if (!dbm::is_result($r)) - return(false); - + if (!dbm::is_result($r)) { + return false; + } $priorities = array(); - foreach ($r AS $line) + foreach ($r AS $line) { $priorities[] = $line["priority"]; - + } // Should not happen - if (count($priorities) == 0) - return(false); - + if (count($priorities) == 0) { + return false; + } $highest_priority = min($priorities); // The highest process is already the slowest one? // Then we quit - if ($highest_priority == PRIORITY_NEGLIGIBLE) - return(false); - + if ($highest_priority == PRIORITY_NEGLIGIBLE) { + return false; + } $high = 0; - foreach ($priorities AS $priority) - if ($priority == $highest_priority) + foreach ($priorities AS $priority) { + if ($priority == $highest_priority) { ++$high; - + } + } logger("Highest priority: ".$highest_priority." Total processes: ".count($priorities)." Count high priority processes: ".$high, LOGGER_DEBUG); $passing_slow = (($high/count($priorities)) > (2/3)); - if ($passing_slow) + if ($passing_slow) { logger("Passing slower processes than priority ".$highest_priority, LOGGER_DEBUG); - - return($passing_slow); + } + return $passing_slow; } /** From 8db079c65e8d3bd72c9cbe739f48b8a5812c14b6 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 5 Jun 2017 06:08:26 +0000 Subject: [PATCH 10/25] Don't always fork the poller. --- boot.php | 13 ++----------- include/cron.php | 5 +++-- include/notifier.php | 6 +++--- include/pubsubpublish.php | 2 +- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/boot.php b/boot.php index f9f8794e5..59011bc64 100644 --- a/boot.php +++ b/boot.php @@ -35,6 +35,7 @@ require_once 'include/features.php'; require_once 'include/identity.php'; require_once 'update.php'; require_once 'include/dbstructure.php'; +require_once 'include/poller.php'; define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Asparagus'); @@ -1095,18 +1096,8 @@ function proc_run($cmd) { return; } - // Checking number of workers - $workers = q("SELECT COUNT(*) AS `workers` FROM `workerqueue` WHERE `executed` > '%s'", dbesc(NULL_DATE)); - - // Get number of allowed number of worker threads - $queues = intval(get_config("system", "worker_queues")); - - if ($queues == 0) { - $queues = 4; - } - // If there are already enough workers running, don't fork another one - if ($workers[0]["workers"] >= $queues) { + if (poller_too_much_workers()) { return; } diff --git a/include/cron.php b/include/cron.php index 3702bf8b3..70b87e969 100644 --- a/include/cron.php +++ b/include/cron.php @@ -246,10 +246,11 @@ function cron_poll_contacts($argc, $argv) { logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]); if (($contact['network'] == NETWORK_FEED) AND ($contact['priority'] <= 3)) { - proc_run(PRIORITY_MEDIUM, 'include/onepoll.php', intval($contact['id'])); + $priority = PRIORITY_MEDIUM; } else { - proc_run(PRIORITY_LOW, 'include/onepoll.php', intval($contact['id'])); + $priority = PRIORITY_LOW; } + proc_run(array('priority' => $priority, 'dont_fork' => true), 'include/onepoll.php', intval($contact['id'])); } } } diff --git a/include/notifier.php b/include/notifier.php index 74cfabb6c..a08057f67 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -498,7 +498,7 @@ function notifier_run(&$argv, &$argc){ } logger("Deliver ".$target_item["guid"]." to ".$contact['url']." via network ".$contact['network'], LOGGER_DEBUG); - proc_run($priority, 'include/delivery.php', $cmd, $item_id, $contact['id']); + proc_run(array('priority' => $priority, 'dont_fork' => true), 'include/delivery.php', $cmd, $item_id, $contact['id']); } } @@ -563,7 +563,7 @@ function notifier_run(&$argv, &$argc){ if ((! $mail) && (! $fsuggest) && (! $followup)) { logger('notifier: delivery agent: '.$rr['name'].' '.$rr['id'].' '.$rr['network'].' '.$target_item["guid"]); - proc_run($priority, 'include/delivery.php', $cmd, $item_id, $rr['id']); + proc_run(array('priority' => $priority, 'dont_fork' => true), 'include/delivery.php', $cmd, $item_id, $rr['id']); } } } @@ -603,7 +603,7 @@ function notifier_run(&$argv, &$argc){ } // Handling the pubsubhubbub requests - proc_run(PRIORITY_HIGH, 'include/pubsubpublish.php'); + proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), 'include/pubsubpublish.php'); } logger('notifier: calling hooks', LOGGER_DEBUG); diff --git a/include/pubsubpublish.php b/include/pubsubpublish.php index 24d7b6963..bb94ea3b2 100644 --- a/include/pubsubpublish.php +++ b/include/pubsubpublish.php @@ -17,7 +17,7 @@ function pubsubpublish_run(&$argv, &$argc){ foreach ($r as $rr) { logger("Publish feed to ".$rr["callback_url"], LOGGER_DEBUG); - proc_run(PRIORITY_HIGH, 'include/pubsubpublish.php', $rr["id"]); + proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), 'include/pubsubpublish.php', $rr["id"]); } } From b86c4d539e8f04f8cbe536d93d6c2fd131a794bf Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 5 Jun 2017 14:59:53 +0000 Subject: [PATCH 11/25] Locking waits now for a shorter period. DB locking is used at other locations as well --- include/dba.php | 17 +++++++++++++++-- include/poller.php | 8 ++++---- include/socgraph.php | 11 ++++------- src/Util/Lock.php | 6 +++--- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/include/dba.php b/include/dba.php index 1fff51f3d..1f428bf46 100644 --- a/include/dba.php +++ b/include/dba.php @@ -816,7 +816,15 @@ class dba { * @return boolean was the lock successful? */ static public function lock($table) { - return self::e("LOCK TABLES `".self::$dbo->escape($table)."` WRITE"); + // See here: https://dev.mysql.com/doc/refman/5.7/en/lock-tables-and-transactions.html + self::e("SET autocommit=0"); + $success = self::e("LOCK TABLES `".self::$dbo->escape($table)."` WRITE"); + if (!$success) { + self::e("SET autocommit=1"); + } else { + self::$in_transaction = true; + } + return $success; } /** @@ -825,7 +833,12 @@ class dba { * @return boolean was the unlock successful? */ static public function unlock() { - return self::e("UNLOCK TABLES"); + // See here: https://dev.mysql.com/doc/refman/5.7/en/lock-tables-and-transactions.html + self::e("COMMIT"); + $success = self::e("UNLOCK TABLES"); + self::e("SET autocommit=1"); + self::$in_transaction = false; + return $success; } /** diff --git a/include/poller.php b/include/poller.php index 3d53be0ab..1de9126d2 100644 --- a/include/poller.php +++ b/include/poller.php @@ -573,7 +573,7 @@ function poller_worker_process() { $highest_priority = 0; if (poller_passing_slow($highest_priority)) { - dba::e('LOCK TABLES `workerqueue` WRITE'); + dba::lock('workerqueue'); // Are there waiting processes with a higher priority than the currently highest? $r = q("SELECT * FROM `workerqueue` @@ -595,7 +595,7 @@ function poller_worker_process() { return $r; } } else { - dba::e('LOCK TABLES `workerqueue` WRITE'); + dba::lock('workerqueue'); } // If there is no result (or we shouldn't pass lower processes) we check without priority limit @@ -605,7 +605,7 @@ function poller_worker_process() { // We only unlock the tables here, when we got no data if (!dbm::is_result($r)) { - dba::e('UNLOCK TABLES'); + dba::unlock(); } return $r; @@ -625,7 +625,7 @@ function poller_claim_process($queue) { $success = dba::update('workerqueue', array('executed' => datetime_convert(), 'pid' => $mypid), array('id' => $queue["id"], 'pid' => 0)); - dba::e('UNLOCK TABLES'); + dba::unlock(); if (!$success) { logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG); diff --git a/include/socgraph.php b/include/socgraph.php index fbac08cc9..7a39e388b 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -1995,10 +1995,11 @@ function get_gcontact_id($contact) { if (in_array($contact["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS))) $contact["url"] = clean_contact_url($contact["url"]); - $r = q("SELECT `id`, `last_contact`, `last_failure`, `network` FROM `gcontact` WHERE `nurl` = '%s' LIMIT 2", + dba::lock('gcontact'); + $r = q("SELECT `id`, `last_contact`, `last_failure`, `network` FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1", dbesc(normalise_link($contact["url"]))); - if ($r) { + if (dbm::is_result($r)) { $gcontact_id = $r[0]["id"]; // Update every 90 days @@ -2036,17 +2037,13 @@ function get_gcontact_id($contact) { $doprobing = in_array($r[0]["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, "")); } } + dba::unlock(); if ($doprobing) { logger("Last Contact: ". $last_contact_str." - Last Failure: ".$last_failure_str." - Checking: ".$contact["url"], LOGGER_DEBUG); proc_run(PRIORITY_LOW, 'include/gprobe.php', bin2hex($contact["url"])); } - if ((dbm::is_result($r)) AND (count($r) > 1) AND ($gcontact_id > 0) AND ($contact["url"] != "")) - q("DELETE FROM `gcontact` WHERE `nurl` = '%s' AND `id` != %d", - dbesc(normalise_link($contact["url"])), - intval($gcontact_id)); - return $gcontact_id; } diff --git a/src/Util/Lock.php b/src/Util/Lock.php index e8011bf59..7cc3472e6 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -57,7 +57,7 @@ class Lock { $memcache = self::memcache(); if (is_object($memcache)) { - $wait_sec = 1; + $wait_sec = 0.2; $cachekey = get_app()->get_hostname().";lock:".$fn_name; do { @@ -77,9 +77,9 @@ class Lock { $got_lock = true; } if (!$got_lock) { - sleep($wait_sec); + usleep($wait_sec * 1000000); } - } while (!$got_lock AND ((time() - $start) < $timeout)); + } while (!$got_lock AND ((time(true) - $start) < $timeout)); return $got_lock; } From 2bff8e302af5d26a7decd78d6e4ab6a181e232e3 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 5 Jun 2017 16:56:21 +0000 Subject: [PATCH 12/25] Removing the lock after the process was removed is better --- include/poller.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/poller.php b/include/poller.php index 1de9126d2..2cf1a15f6 100644 --- a/include/poller.php +++ b/include/poller.php @@ -752,11 +752,11 @@ function poller_run_cron() { if (array_search(__file__,get_included_files())===0){ poller_run($_SERVER["argv"],$_SERVER["argc"]); - Lock::remove('poller_worker'); - poller_unclaim_process(); get_app()->end_process(); + Lock::remove('poller_worker'); + killme(); } From 2b04865cdbb7c7516b188f786af739d1011b4cbb Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 5 Jun 2017 22:41:33 +0000 Subject: [PATCH 13/25] We found the handbrake ... --- include/poller.php | 21 ++++++++------------- src/Util/Lock.php | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/include/poller.php b/include/poller.php index 2cf1a15f6..4bd1ee9e4 100644 --- a/include/poller.php +++ b/include/poller.php @@ -89,21 +89,16 @@ function poller_run($argv, $argc){ continue; } - // To avoid the quitting of multiple pollers we serialize the next check - if (!Lock::set('poller_worker')) { - logger('Cannot get a lock, retrying.', LOGGER_DEBUG); - poller_unclaim_process(); - continue; + // To avoid the quitting of multiple pollers only one poller at a time will execute the check + if (Lock::set('poller_worker', 0)) { + // Count active workers and compare them with a maximum value that depends on the load + if (poller_too_much_workers()) { + logger('Active worker limit reached, quitting.', LOGGER_DEBUG); + return; + } + Lock::remove('poller_worker'); } - // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers()) { - logger('Active worker limit reached, quitting.', LOGGER_DEBUG); - return; - } - - Lock::remove('poller_worker'); - // Check free memory if ($a->min_memory_reached()) { logger('Memory limit reached, quitting.', LOGGER_DEBUG); diff --git a/src/Util/Lock.php b/src/Util/Lock.php index 7cc3472e6..63f9b5f97 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -79,7 +79,7 @@ class Lock { if (!$got_lock) { usleep($wait_sec * 1000000); } - } while (!$got_lock AND ((time(true) - $start) < $timeout)); + } while (!$got_lock AND ((time() - $start) < $timeout)); return $got_lock; } From ba7b4fddea9cf346916a23455f2273c97cf9e7c9 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 04:00:24 +0000 Subject: [PATCH 14/25] memory check is now also only done once in a while --- include/poller.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/poller.php b/include/poller.php index 4bd1ee9e4..68c5acbcb 100644 --- a/include/poller.php +++ b/include/poller.php @@ -96,13 +96,13 @@ function poller_run($argv, $argc){ logger('Active worker limit reached, quitting.', LOGGER_DEBUG); return; } - Lock::remove('poller_worker'); - } - // Check free memory - if ($a->min_memory_reached()) { - logger('Memory limit reached, quitting.', LOGGER_DEBUG); - return; + // Check free memory + if ($a->min_memory_reached()) { + logger('Memory limit reached, quitting.', LOGGER_DEBUG); + return; + } + Lock::remove('poller_worker'); } // finally the work will be done From 5dfa513b62d37395039ddf60aecc867028b8f91f Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 04:22:18 +0000 Subject: [PATCH 15/25] Only wait when you have a timout value at all --- src/Util/Lock.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Util/Lock.php b/src/Util/Lock.php index 63f9b5f97..af15e106d 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -76,7 +76,7 @@ class Lock { $memcache->set($cachekey, getmypid(), MEMCACHE_COMPRESSED, 300); $got_lock = true; } - if (!$got_lock) { + if (!$got_lock AND ($timeout > 0)) { usleep($wait_sec * 1000000); } } while (!$got_lock AND ((time() - $start) < $timeout)); @@ -112,7 +112,7 @@ class Lock { dba::unlock(); - if (!$got_lock) { + if (!$got_lock AND ($timeout > 0)) { sleep($wait_sec); } } while (!$got_lock AND ((time() - $start) < $timeout)); From e00105d649d4a8e3ff89820563470cdd5d742ab9 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 11:59:38 +0000 Subject: [PATCH 16/25] Don't fork a new worker --- include/queue.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/queue.php b/include/queue.php index b21bf676b..6b42cf31e 100644 --- a/include/queue.php +++ b/include/queue.php @@ -27,7 +27,7 @@ function queue_run(&$argv, &$argc){ logger('queue: start'); // Handling the pubsubhubbub requests - proc_run(PRIORITY_HIGH,'include/pubsubpublish.php'); + proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), 'include/pubsubpublish.php'); $r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue` INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id` @@ -50,7 +50,7 @@ function queue_run(&$argv, &$argc){ if (dbm::is_result($r)) { foreach ($r as $q_item) { logger('Call queue for id '.$q_item['id']); - proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']); + proc_run(array('priority' => PRIORITY_LOW, 'dont_fork' => true), "include/queue.php", $q_item['id']); } } return; From 8d13751d40d40d1d348584ae5ff803096e09f3af Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 12:07:27 +0000 Subject: [PATCH 17/25] Increased database version --- boot.php | 2 +- database.sql | 2 +- include/poller.php | 5 ++--- update.php | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/boot.php b/boot.php index 59011bc64..b1f956fb2 100644 --- a/boot.php +++ b/boot.php @@ -41,7 +41,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Asparagus'); define ( 'FRIENDICA_VERSION', '3.5.2-rc' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1227 ); +define ( 'DB_UPDATE_VERSION', 1228 ); /** * @brief Constant with a HTML line break. diff --git a/database.sql b/database.sql index 4a5946ef3..2c1326491 100644 --- a/database.sql +++ b/database.sql @@ -580,7 +580,7 @@ CREATE TABLE IF NOT EXISTS `locks` ( `id` int(11) NOT NULL auto_increment, `name` varchar(128) NOT NULL DEFAULT '', `locked` tinyint(1) NOT NULL DEFAULT 0, - `created` datetime DEFAULT '0001-01-01 00:00:00', + `pid` int(10) unsigned NOT NULL DEFAULT 0, PRIMARY KEY(`id`) ) DEFAULT COLLATE utf8mb4_general_ci; diff --git a/include/poller.php b/include/poller.php index 68c5acbcb..b08d98ca7 100644 --- a/include/poller.php +++ b/include/poller.php @@ -461,8 +461,7 @@ function poller_too_much_workers() { dba::close($processes); } dba::close($entries); - - $processlist = implode(', ', $listitem); + $processlist = ' ('.implode(', ', $listitem).')'; $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` <= '%s'", dbesc(NULL_DATE)); $entries = $s[0]["total"]; @@ -481,7 +480,7 @@ function poller_too_much_workers() { } } - logger("Load: ".$load."/".$maxsysload." - processes: ".$active."/".$entries." (".$processlist.") - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG); + logger("Load: ".$load."/".$maxsysload." - processes: ".$active."/".$entries.$processlist." - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG); // Are there fewer workers running as possible? Then fork a new one. if (!Config::get("system", "worker_dont_fork") AND ($queues > ($active + 1)) AND ($entries > 1)) { diff --git a/update.php b/update.php index 7561a9af1..76620a48a 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ Date: Tue, 6 Jun 2017 17:25:28 +0000 Subject: [PATCH 18/25] Added documentation and renamed function --- doc/database/db_locks.md | 12 +++++------ src/Util/Lock.php | 44 ++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/doc/database/db_locks.md b/doc/database/db_locks.md index 00556dd95..004ebd1c1 100644 --- a/doc/database/db_locks.md +++ b/doc/database/db_locks.md @@ -1,11 +1,11 @@ Table locks =========== -| Field | Description | Type | Null | Key | Default | Extra | -|---------|------------------|--------------|------|-----|---------------------|----------------| -| id | sequential ID | int(11) | NO | PRI | NULL | auto_increment | -| name | | varchar(128) | NO | | | | -| locked | | tinyint(1) | NO | | 0 | | -| created | | datetime | YES | | 0001-01-01 00:00:00 | | +| Field | Description | Type | Null | Key | Default | Extra | +|---------|------------------|------------------|------|-----|---------------------|----------------| +| id | sequential ID | int(11) | NO | PRI | NULL | auto_increment | +| name | | varchar(128) | NO | | | | +| locked | | tinyint(1) | NO | | 0 | | +| pid | | int(10) unsigned | NO | | 0 | | Return to [database documentation](help/database) diff --git a/src/Util/Lock.php b/src/Util/Lock.php index af15e106d..ca327de1e 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -18,30 +18,30 @@ use dbm; */ class Lock { /** - * @brief Check for memcache and open a connection if configured - * - * @return object|boolean The memcache object - or "false" if not successful - */ - public static function memcache() { - if (!function_exists('memcache_connect')) { - return false; - } + * @brief Check for memcache and open a connection if configured + * + * @return object|boolean The memcache object - or "false" if not successful + */ + private static function connect_memcache() { + if (!function_exists('memcache_connect')) { + return false; + } - if (!Config::get('system', 'memcache')) { - return false; - } + if (!Config::get('system', 'memcache')) { + return false; + } - $memcache_host = Config::get('system', 'memcache_host', '127.0.0.1'); - $memcache_port = Config::get('system', 'memcache_port', 11211); + $memcache_host = Config::get('system', 'memcache_host', '127.0.0.1'); + $memcache_port = Config::get('system', 'memcache_port', 11211); - $memcache = new Memcache; + $memcache = new Memcache; - if (!$memcache->connect($memcache_host, $memcache_port)) { - return false; - } + if (!$memcache->connect($memcache_host, $memcache_port)) { + return false; + } - return $memcache; - } + return $memcache; + } /** * @brief Sets a lock for a given name @@ -55,7 +55,7 @@ class Lock { $got_lock = false; $start = time(); - $memcache = self::memcache(); + $memcache = self::connect_memcache(); if (is_object($memcache)) { $wait_sec = 0.2; $cachekey = get_app()->get_hostname().";lock:".$fn_name; @@ -126,7 +126,7 @@ class Lock { * @param string $fn_name Name of the lock */ public static function remove($fn_name) { - $memcache = self::memcache(); + $memcache = self::connect_memcache(); if (is_object($memcache)) { $cachekey = get_app()->get_hostname().";lock:".$fn_name; $lock = $memcache->get($cachekey); @@ -147,7 +147,7 @@ class Lock { * @brief Removes all lock that were set by us */ public static function removeAll() { - $memcache = self::memcache(); + $memcache = self::connect_memcache(); if (is_object($memcache)) { // We cannot delete all cache entries, but this doesn't matter with memcache return; From 929f518e5c67b7424e9249babcc7648daa2cd3fd Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 17:38:58 +0000 Subject: [PATCH 19/25] Added documentation --- include/dbclean.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/dbclean.php b/include/dbclean.php index 024b845f7..8da78d64b 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -35,6 +35,19 @@ function dbclean_run(&$argv, &$argc) { /** * @brief Remove orphaned database entries + * @param integer $stage What should be deleted? + * + * Values for $stage: + * ------------------ + * 1: Old global item entries from item table without user copy. + * 2: Items without parents. + * 3: Orphaned data from thread table. + * 4: Orphaned data from notify table. + * 5: Orphaned data from notify-threads table. + * 6: Orphaned data from sign table. + * 7: Orphaned data from term table. + * 8: Expired threads. + * 9: Old global item entries from expired threads */ function remove_orphans($stage = 0) { global $db; From 611d3e3f5d781a3d3bc2d8cd3c1360a3297a09f5 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 17:41:01 +0000 Subject: [PATCH 20/25] Added documentation --- doc/database/db_locks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/database/db_locks.md b/doc/database/db_locks.md index 004ebd1c1..4de6fbf96 100644 --- a/doc/database/db_locks.md +++ b/doc/database/db_locks.md @@ -6,6 +6,6 @@ Table locks | id | sequential ID | int(11) | NO | PRI | NULL | auto_increment | | name | | varchar(128) | NO | | | | | locked | | tinyint(1) | NO | | 0 | | -| pid | | int(10) unsigned | NO | | 0 | | +| pid | Process ID | int(10) unsigned | NO | | 0 | | Return to [database documentation](help/database) From e3d5dcf049b4e772ae05a924f4e1d045a3480006 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 17:42:47 +0000 Subject: [PATCH 21/25] Beware of camels --- src/Util/Lock.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Util/Lock.php b/src/Util/Lock.php index ca327de1e..26ccdc9dd 100644 --- a/src/Util/Lock.php +++ b/src/Util/Lock.php @@ -22,7 +22,7 @@ class Lock { * * @return object|boolean The memcache object - or "false" if not successful */ - private static function connect_memcache() { + private static function connectMemcache() { if (!function_exists('memcache_connect')) { return false; } @@ -55,7 +55,7 @@ class Lock { $got_lock = false; $start = time(); - $memcache = self::connect_memcache(); + $memcache = self::connectMemcache(); if (is_object($memcache)) { $wait_sec = 0.2; $cachekey = get_app()->get_hostname().";lock:".$fn_name; @@ -126,7 +126,7 @@ class Lock { * @param string $fn_name Name of the lock */ public static function remove($fn_name) { - $memcache = self::connect_memcache(); + $memcache = self::connectMemcache(); if (is_object($memcache)) { $cachekey = get_app()->get_hostname().";lock:".$fn_name; $lock = $memcache->get($cachekey); @@ -147,7 +147,7 @@ class Lock { * @brief Removes all lock that were set by us */ public static function removeAll() { - $memcache = self::connect_memcache(); + $memcache = self::connectMemcache(); if (is_object($memcache)) { // We cannot delete all cache entries, but this doesn't matter with memcache return; From 4e748668c6862cd4ab37570b77c4f84c1ab855d3 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 20:10:47 +0000 Subject: [PATCH 22/25] Spaces --- include/poller.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/poller.php b/include/poller.php index a19d9dec8..fcabe5d8e 100644 --- a/include/poller.php +++ b/include/poller.php @@ -392,7 +392,7 @@ function poller_kill_stale_workers() { $pid["priority"] = PRIORITY_MEDIUM; } - // Define the maximum durations + // Define the maximum durations $max_duration_defaults = array(PRIORITY_CRITICAL => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 360); $max_duration = $max_duration_defaults[$pid["priority"]]; From 9c46971aa550e852788b5c5bae66aa660e191faa Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 6 Jun 2017 22:18:42 +0000 Subject: [PATCH 23/25] The check for blocking processes is deactivated until further checks were performed --- src/App.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/App.php b/src/App.php index d671c5f1a..26cfcaadb 100644 --- a/src/App.php +++ b/src/App.php @@ -797,6 +797,8 @@ class App { * @return bool Is the limit reached? */ function max_processes_reached() { + // Deactivated, needs more investigating if this check really makes sense + return false; if ($this->is_backend()) { $process = 'backend'; From 36630daca6569eed8f71dab6663e4fce113aa6ff Mon Sep 17 00:00:00 2001 From: Tobias Diekershoff Date: Wed, 7 Jun 2017 07:36:04 +0200 Subject: [PATCH 24/25] added missing dash in version --- VERSION | 2 +- boot.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 3fec5bc90..b35d44161 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.5.3dev +3.5.3-dev diff --git a/boot.php b/boot.php index 5f83cb3e4..9428cf5a3 100644 --- a/boot.php +++ b/boot.php @@ -38,7 +38,7 @@ require_once 'include/dbstructure.php'; define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Asparagus'); -define ( 'FRIENDICA_VERSION', '3.5.3dev' ); +define ( 'FRIENDICA_VERSION', '3.5.3-dev' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); define ( 'DB_UPDATE_VERSION', 1227 ); From 2ad784d37ae88338f9bc610cd527c72fad052aa0 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 7 Jun 2017 05:42:30 +0000 Subject: [PATCH 25/25] Speeded up calling "proc_run" --- database.sql | 5 +++-- include/dbstructure.php | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/database.sql b/database.sql index 2c1326491..7f7e975e7 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ --- Friendica 3.5.2-rc (Asparagus) --- DB_UPDATE_VERSION 1227 +-- Friendica 3.5.3dev (Asparagus) +-- DB_UPDATE_VERSION 1228 -- ------------------------------------------ @@ -1116,6 +1116,7 @@ CREATE TABLE IF NOT EXISTS `workerqueue` ( `executed` datetime NOT NULL DEFAULT '0001-01-01 00:00:00', PRIMARY KEY(`id`), INDEX `pid` (`pid`), + INDEX `parameter` (`parameter`(192)), INDEX `priority_created` (`priority`,`created`) ) DEFAULT COLLATE utf8mb4_general_ci; diff --git a/include/dbstructure.php b/include/dbstructure.php index 844930763..161df46f6 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -1743,6 +1743,7 @@ function db_definition() { "indexes" => array( "PRIMARY" => array("id"), "pid" => array("pid"), + "parameter" => array("parameter(192)"), "priority_created" => array("priority", "created"), ) );