From b3c8ff836d0844d6ae18a471157d03e861b1ec3c Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 2 Jul 2016 08:54:57 +0200 Subject: [PATCH 1/2] Backend/Frontend detection when checking for overload --- boot.php | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- index.php | 45 ++++++++------------------ 2 files changed, 105 insertions(+), 35 deletions(-) diff --git a/boot.php b/boot.php index 42767845c2..2db7d49d00 100644 --- a/boot.php +++ b/boot.php @@ -475,6 +475,7 @@ class App { public $performance = array(); public $callstack = array(); public $theme_info = array(); + public $backend = true; public $nav_sel; @@ -1111,6 +1112,80 @@ class App { return($this->is_friendica_app); } + /** + * @brief Checks if the site is called via a backend process + * + * This isn't a perfect solution. But we need this check very early. + * So we cannot wait until the modules are loaded. + * + * @return bool Is it a known backend? + */ + function is_backend() { + $backend = array(); + $backend[] = "_well_known"; + $backend[] = "api"; + $backend[] = "dfrn_notify"; + $backend[] = "dfrn_poll"; + $backend[] = "fetch"; + $backend[] = "hcard"; + $backend[] = "hostxrd"; + $backend[] = "nodeinfo"; + $backend[] = "noscrape"; + $backend[] = "p"; + $backend[] = "photo"; + $backend[] = "photos"; + $backend[] = "poco"; + $backend[] = "post"; + $backend[] = "proxy"; + $backend[] = "pubsub"; + $backend[] = "pubsubhubbub"; + $backend[] = "receive"; + $backend[] = "rsd_xml"; + $backend[] = "salmon"; + $backend[] = "statistics_json"; + $backend[] = "xrd"; + + if (in_array($this->module, $backend)) + return(true); + else + return($this->backend); + } + + /** + * @brief Checks if the maximum number of database processes is reached + * + * @return bool Is the limit reached? + */ + function max_processes_reached() { + + // Is the function called statically? + if (!is_object($this)) + return(self::$a->max_processes_reached()); + + if ($this->is_backend()) { + $process = "backend"; + $max_processes = get_config('system', 'max_processes_backend'); + if (intval($max_processes) == 0) + $max_processes = 5; + } else { + $process = "frontend"; + $max_processes = get_config('system', 'max_processes_frontend'); + if (intval($max_processes) == 0) + $max_processes = 20; + } + + $processlist = dbm::processlist(); + if ($processlist["list"] != "") { + logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); + + if ($processlist["amount"] > $max_processes) { + logger("Processcheck: Maximum number of processes for ".$process." tasks (".$max_processes.") reached.", LOGGER_DEBUG); + return true; + } + } + return false; + } + /** * @brief Checks if the maximum load is reached * @@ -1118,14 +1193,26 @@ class App { */ function maxload_reached() { - $maxsysload = intval(get_config('system', 'maxloadavg')); - if ($maxsysload < 1) - $maxsysload = 50; + // Is the function called statically? + if (!is_object($this)) + return(self::$a->maxload_reached()); + + if ($this->is_backend()) { + $process = "backend"; + $maxsysload = intval(get_config('system', 'maxloadavg')); + if ($maxsysload < 1) + $maxsysload = 50; + } else { + $process = "frontend"; + $maxsysload = intval(get_config('system','maxloadavg_frontend')); + if ($maxsysload < 1) + $maxsysload = 50; + } $load = current_load(); if ($load) { if (intval($load) > $maxsysload) { - logger('system: load '.$load.' too high.'); + logger('system: load '.$load.' for '.$process.' tasks ('.$maxsysload.') too high.'); return true; } } diff --git a/index.php b/index.php index 943763a511..d2d3ecec3a 100644 --- a/index.php +++ b/index.php @@ -19,6 +19,10 @@ require_once('object/BaseObject.php'); $a = new App; BaseObject::set_app($a); +// We assume that the index.php is called by a frontend process +// The value is set to "true" by default in boot.php +$a->backend = false; + /** * * Load the configuration file which contains our DB credentials. @@ -54,36 +58,12 @@ if(!$install) { load_config('config'); load_config('system'); - $processlist = dbm::processlist(); - if ($processlist["list"] != "") { - - logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); - - $max_processes = get_config('system', 'max_processes_frontend'); - if (intval($max_processes) == 0) - $max_processes = 20; - - if ($processlist["amount"] > $max_processes) { - logger("Processcheck: Maximum number of processes for frontend tasks (".$max_processes.") reached.", LOGGER_DEBUG); - system_unavailable(); - } + if ($a->max_processes_reached() OR $a->maxload_reached()) { + header($_SERVER["SERVER_PROTOCOL"].' 503 Service Temporarily Unavailable'); + header('Retry-After: 300'); + die("System is currently unavailable. Please try again later"); } - $maxsysload_frontend = intval(get_config('system','maxloadavg_frontend')); - if($maxsysload_frontend < 1) - $maxsysload_frontend = 50; - - $load = current_load(); - if($load) { - if($load > $maxsysload_frontend) { - logger('system: load ' . $load . ' too high. Service Temporarily Unavailable.'); - header($_SERVER["SERVER_PROTOCOL"].' 503 Service Temporarily Unavailable'); - header('Retry-After: 300'); - die("System is currently unavailable. Please try again later"); - } - } - - if (get_config('system','force_ssl') AND ($a->get_scheme() == "http") AND (intval(get_config('system','ssl_policy')) == SSL_POLICY_FULL) AND (substr($a->get_baseurl(), 0, 8) == "https://")) { @@ -114,9 +94,12 @@ load_translation_table($lang); * */ -$stamp1 = microtime(true); -session_start(); -$a->save_timestamp($stamp1, "parser"); +// Exclude the backend processes from the session management +if (!$a->is_backend()) { + $stamp1 = microtime(true); + session_start(); + $a->save_timestamp($stamp1, "parser"); +} /** * Language was set earlier, but we can over-ride it in the session. From c549ae939cd424c18052efd31f2d25ff9073f05f Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 2 Jul 2016 09:31:28 +0200 Subject: [PATCH 2/2] Added the new function to the poller. --- include/dba.php | 1 + include/poller.php | 29 +++++------------------------ index.php | 1 - 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/include/dba.php b/include/dba.php index cae045d874..6ce095fc65 100644 --- a/include/dba.php +++ b/include/dba.php @@ -1,4 +1,5 @@ $max_processes) { - logger("Processcheck: Maximum number of processes for backend tasks (".$max_processes.") reached.", LOGGER_DEBUG); - return; - } - } + if ($a->max_processes_reached()) + return; if (poller_max_connections_reached()) return; @@ -74,16 +62,9 @@ function poller_run(&$argv, &$argc){ while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `created` LIMIT 1")) { - // Log the type of database processes - $processlist = dbm::processlist(); - if ($processlist["amount"] != "") { - logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); - - if ($processlist["amount"] > $max_processes) { - logger("Processcheck: Maximum number of processes for backend tasks (".$max_processes.") reached.", LOGGER_DEBUG); - return; - } - } + // Constantly check the number of parallel database processes + if ($a->max_processes_reached()) + return; // Constantly check the number of available database connections to let the frontend be accessible at any time if (poller_max_connections_reached()) diff --git a/index.php b/index.php index d2d3ecec3a..cde267cd19 100644 --- a/index.php +++ b/index.php @@ -45,7 +45,6 @@ $install = ((file_exists('.htconfig.php') && filesize('.htconfig.php')) ? false */ require_once("include/dba.php"); -require_once("include/dbm.php"); if(!$install) { $db = new dba($db_host, $db_user, $db_pass, $db_data, $install);