Merge pull request #2658 from annando/1607-maxload

Central Backend/Frontend detection when checking for overload

boot.php

@@ -475,6 +475,7 @@ class App {
 	public	$performance = array();
 	public	$callstack = array();
 	public	$theme_info = array();
+	public  $backend = true;
 
 	public $nav_sel;
 
@@ -1111,6 +1112,80 @@ class App {
 		return($this->is_friendica_app);
 	}
 
+	/**
+	 * @brief Checks if the site is called via a backend process
+	 *
+	 * This isn't a perfect solution. But we need this check very early.
+	 * So we cannot wait until the modules are loaded.
+	 *
+	 * @return bool Is it a known backend?
+	 */
+	function is_backend() {
+		$backend = array();
+		$backend[] = "_well_known";
+		$backend[] = "api";
+		$backend[] = "dfrn_notify";
+		$backend[] = "dfrn_poll";
+		$backend[] = "fetch";
+		$backend[] = "hcard";
+		$backend[] = "hostxrd";
+		$backend[] = "nodeinfo";
+		$backend[] = "noscrape";
+		$backend[] = "p";
+		$backend[] = "photo";
+		$backend[] = "photos";
+		$backend[] = "poco";
+		$backend[] = "post";
+		$backend[] = "proxy";
+		$backend[] = "pubsub";
+		$backend[] = "pubsubhubbub";
+		$backend[] = "receive";
+		$backend[] = "rsd_xml";
+		$backend[] = "salmon";
+		$backend[] = "statistics_json";
+		$backend[] = "xrd";
+
+		if (in_array($this->module, $backend))
+			return(true);
+		else
+			return($this->backend);
+	}
+
+	/**
+	 * @brief Checks if the maximum number of database processes is reached
+	 *
+	 * @return bool Is the limit reached?
+	 */
+	function max_processes_reached() {
+
+		// Is the function called statically?
+		if (!is_object($this))
+			return(self::$a->max_processes_reached());
+
+		if ($this->is_backend()) {
+			$process = "backend";
+			$max_processes = get_config('system', 'max_processes_backend');
+			if (intval($max_processes) == 0)
+				$max_processes = 5;
+		} else {
+			$process = "frontend";
+			$max_processes = get_config('system', 'max_processes_frontend');
+			if (intval($max_processes) == 0)
+				$max_processes = 20;
+		}
+
+		$processlist = dbm::processlist();
+		if ($processlist["list"] != "") {
+			logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG);
+
+			if ($processlist["amount"] > $max_processes) {
+				logger("Processcheck: Maximum number of processes for ".$process." tasks (".$max_processes.") reached.", LOGGER_DEBUG);
+				return true;
+			}
+		}
+		return false;
+	}
+
 	/**
 	 * @brief Checks if the maximum load is reached
 	 *
@@ -1118,14 +1193,26 @@ class App {
 	 */
 	function maxload_reached() {
 
-		$maxsysload = intval(get_config('system', 'maxloadavg'));
+		// Is the function called statically?
-		if ($maxsysload < 1)
+		if (!is_object($this))
-			$maxsysload = 50;
+			return(self::$a->maxload_reached());
+
+		if ($this->is_backend()) {
+			$process = "backend";
+			$maxsysload = intval(get_config('system', 'maxloadavg'));
+			if ($maxsysload < 1)
+				$maxsysload = 50;
+		} else {
+			$process = "frontend";
+			$maxsysload = intval(get_config('system','maxloadavg_frontend'));
+			if ($maxsysload < 1)
+				$maxsysload = 50;
+		}
 
 		$load = current_load();
 		if ($load) {
 			if (intval($load) > $maxsysload) {
-				logger('system: load '.$load.' too high.');
+				logger('system: load '.$load.' for '.$process.' tasks ('.$maxsysload.') too high.');
 				return true;
 			}
 		}

include/dba.php

@@ -1,4 +1,5 @@
 <?php
+require_once("dbm.php");
 
 # if PDO is avaible for mysql, use the new database abstraction
 # TODO: PDO is disabled for release 3.3. We need to investigate why

include/poller.php

@@ -11,7 +11,6 @@ if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) {
 }
 
 require_once("boot.php");
-require_once("dbm.php");
 
 function poller_run(&$argv, &$argc){
 	global $a, $db;
@@ -27,19 +26,8 @@ function poller_run(&$argv, &$argc){
 		unset($db_host, $db_user, $db_pass, $db_data);
 	};
 
-	$max_processes = get_config('system', 'max_processes_backend');
+	if ($a->max_processes_reached())
-	if (intval($max_processes) == 0)
+		return;
-		$max_processes = 5;
-
-	$processlist = dbm::processlist();
-	if ($processlist["list"] != "") {
-		logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG);
-
-		if ($processlist["amount"] > $max_processes) {
-			logger("Processcheck: Maximum number of processes for backend tasks (".$max_processes.") reached.", LOGGER_DEBUG);
-			return;
-		}
-	}
 
 	if (poller_max_connections_reached())
 		return;
@@ -74,16 +62,9 @@ function poller_run(&$argv, &$argc){
 
 	while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `created` LIMIT 1")) {
 
-		// Log the type of database processes
+		// Constantly check the number of parallel database processes
-		$processlist = dbm::processlist();
+		if ($a->max_processes_reached())
-		if ($processlist["amount"] != "") {
+			return;
-			logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG);
-
-			if ($processlist["amount"] > $max_processes) {
-				logger("Processcheck: Maximum number of processes for backend tasks (".$max_processes.") reached.", LOGGER_DEBUG);
-				return;
-			}
-		}
 
 		// Constantly check the number of available database connections to let the frontend be accessible at any time
 		if (poller_max_connections_reached())

index.php

@@ -19,6 +19,10 @@ require_once('object/BaseObject.php');
 $a = new App;
 BaseObject::set_app($a);
 
+// We assume that the index.php is called by a frontend process
+// The value is set to "true" by default in boot.php
+$a->backend = false;
+
 /**
  *
  * Load the configuration file which contains our DB credentials.
@@ -41,7 +45,6 @@ $install = ((file_exists('.htconfig.php') && filesize('.htconfig.php')) ? false
  */
 
 require_once("include/dba.php");
-require_once("include/dbm.php");
 
 if(!$install) {
 	$db = new dba($db_host, $db_user, $db_pass, $db_data, $install);
@@ -54,36 +57,12 @@ if(!$install) {
 	load_config('config');
 	load_config('system');
 
-	$processlist = dbm::processlist();
+	if ($a->max_processes_reached() OR $a->maxload_reached()) {
-	if ($processlist["list"] != "") {
+		header($_SERVER["SERVER_PROTOCOL"].' 503 Service Temporarily Unavailable');
-
+		header('Retry-After: 300');
-		logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG);
+		die("System is currently unavailable. Please try again later");
-
-		$max_processes = get_config('system', 'max_processes_frontend');
-		if (intval($max_processes) == 0)
-			$max_processes = 20;
-
-		if ($processlist["amount"] > $max_processes) {
-			logger("Processcheck: Maximum number of processes for frontend tasks (".$max_processes.") reached.", LOGGER_DEBUG);
-			system_unavailable();
-		}
 	}
 
-	$maxsysload_frontend = intval(get_config('system','maxloadavg_frontend'));
-	if($maxsysload_frontend < 1)
-		$maxsysload_frontend = 50;
-
-	$load = current_load();
-	if($load) {
-		if($load > $maxsysload_frontend) {
-			logger('system: load ' . $load . ' too high. Service Temporarily Unavailable.');
-			header($_SERVER["SERVER_PROTOCOL"].' 503 Service Temporarily Unavailable');
-			header('Retry-After: 300');
-			die("System is currently unavailable. Please try again later");
-		}
-	}
-
-
 	if (get_config('system','force_ssl') AND ($a->get_scheme() == "http") AND
 		(intval(get_config('system','ssl_policy')) == SSL_POLICY_FULL) AND
 		(substr($a->get_baseurl(), 0, 8) == "https://")) {
@@ -114,9 +93,12 @@ load_translation_table($lang);
  *
  */
 
-$stamp1 = microtime(true);
+// Exclude the backend processes from the session management
-session_start();
+if (!$a->is_backend()) {
-$a->save_timestamp($stamp1, "parser");
+	$stamp1 = microtime(true);
+	session_start();
+	$a->save_timestamp($stamp1, "parser");
+}
 
 /**
  * Language was set earlier, but we can over-ride it in the session.