From da212a28a2b483396495269a0adb4f759884980c Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 5 Nov 2017 10:33:46 +0000 Subject: [PATCH 1/8] New worker class that does all the work --- boot.php | 3 +- include/poller.php | 864 +------------------------------------- index.php | 5 +- mod/worker.php | 15 +- src/Core/Worker.php | 993 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1014 insertions(+), 866 deletions(-) create mode 100644 src/Core/Worker.php diff --git a/boot.php b/boot.php index cdd71719a6..bf38c911cc 100644 --- a/boot.php +++ b/boot.php @@ -22,6 +22,7 @@ require_once(__DIR__ . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'a use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; use Friendica\Core\Config; use Friendica\Util\Lock; @@ -1127,7 +1128,7 @@ function proc_run($cmd) { } // If there are already enough workers running, don't fork another one - $quit = poller_too_much_workers(); + $quit = Worker::tooMuchWorkers(); Lock::remove('poller_worker'); if ($quit) { diff --git a/include/poller.php b/include/poller.php index b8f0d7189f..a18d10b3c6 100644 --- a/include/poller.php +++ b/include/poller.php @@ -1,9 +1,7 @@ maxload_reached()) { - logger('Pre check: maximum load reached, quitting.', LOGGER_DEBUG); - return; - } + $run_cron = (($argc <= 1) || ($argv[1] != "no_cron")); - // We now start the process. This is done after the load check since this could increase the load. - $a->start_process(); - - // Kill stale processes every 5 minutes - $last_cleanup = Config::get('system', 'poller_last_cleaned', 0); - if (time() > ($last_cleanup + 300)) { - Config::set('system', 'poller_last_cleaned', time()); - poller_kill_stale_workers(); - } - - // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers()) { - logger('Pre check: Active worker limit reached, quitting.', LOGGER_DEBUG); - return; - } - - // Do we have too few memory? - if ($a->min_memory_reached()) { - logger('Pre check: Memory limit reached, quitting.', LOGGER_DEBUG); - return; - } - - // Possibly there are too much database connections - if (poller_max_connections_reached()) { - logger('Pre check: maximum connections reached, quitting.', LOGGER_DEBUG); - return; - } - - // Possibly there are too much database processes that block the system - if ($a->max_processes_reached()) { - logger('Pre check: maximum processes reached, quitting.', LOGGER_DEBUG); - return; - } - - // Now we start additional cron processes if we should do so - if (($argc <= 1) || ($argv[1] != "no_cron")) { - poller_run_cron(); - } - - $starttime = time(); - - // We fetch the next queue entry that is about to be executed - while ($r = poller_worker_process($passing_slow)) { - - // When we are processing jobs with a lower priority, we don't refetch new jobs - // Otherwise fast jobs could wait behind slow ones and could be blocked. - $refetched = $passing_slow; - - foreach ($r AS $entry) { - // Assure that the priority is an integer value - $entry['priority'] = (int)$entry['priority']; - - // The work will be done - if (!poller_execute($entry)) { - logger('Process execution failed, quitting.', LOGGER_DEBUG); - return; - } - - // If possible we will fetch new jobs for this worker - if (!$refetched && Lock::set('poller_worker_process', 0)) { - $stamp = (float)microtime(true); - $refetched = find_worker_processes($passing_slow); - $poller_db_duration += (microtime(true) - $stamp); - Lock::remove('poller_worker_process'); - } - } - - // To avoid the quitting of multiple pollers only one poller at a time will execute the check - if (Lock::set('poller_worker', 0)) { - $stamp = (float)microtime(true); - // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers()) { - logger('Active worker limit reached, quitting.', LOGGER_DEBUG); - return; - } - - // Check free memory - if ($a->min_memory_reached()) { - logger('Memory limit reached, quitting.', LOGGER_DEBUG); - return; - } - Lock::remove('poller_worker'); - $poller_db_duration += (microtime(true) - $stamp); - } - - // Quit the poller once every 5 minutes - if (time() > ($starttime + 300)) { - logger('Process lifetime reached, quitting.', LOGGER_DEBUG); - return; - } - } - logger("Couldn't select a workerqueue entry, quitting.", LOGGER_DEBUG); + Worker::processQueue($run_cron); + return; } -/** - * @brief Returns the number of non executed entries in the worker queue - * - * @return integer Number of non executed entries in the worker queue - */ -function poller_total_entries() { - $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` <= '%s' AND NOT `done`", dbesc(NULL_DATE)); - if (dbm::is_result($s)) { - return $s[0]["total"]; - } else { - return 0; - } -} +if (array_search(__file__, get_included_files()) === 0) { + poller_run($_SERVER["argv"], $_SERVER["argc"]); -/** - * @brief Returns the highest priority in the worker queue that isn't executed - * - * @return integer Number of active poller processes - */ -function poller_highest_priority() { - $s = q("SELECT `priority` FROM `workerqueue` WHERE `executed` <= '%s' AND NOT `done` ORDER BY `priority` LIMIT 1", dbesc(NULL_DATE)); - if (dbm::is_result($s)) { - return $s[0]["priority"]; - } else { - return 0; - } -} - -/** - * @brief Returns if a process with the given priority is running - * - * @param integer $priority The priority that should be checked - * - * @return integer Is there a process running with that priority? - */ -function poller_process_with_priority_active($priority) { - $s = q("SELECT `id` FROM `workerqueue` WHERE `priority` <= %d AND `executed` > '%s' AND NOT `done` LIMIT 1", - intval($priority), dbesc(NULL_DATE)); - return dbm::is_result($s); -} - -/** - * @brief Execute a worker entry - * - * @param array $queue Workerqueue entry - * - * @return boolean "true" if further processing should be stopped - */ -function poller_execute($queue) { - global $poller_db_duration, $poller_last_update; - - $a = get_app(); - - $mypid = getmypid(); - - // Quit when in maintenance - if (Config::get('system', 'maintenance', true)) { - logger("Maintenance mode - quit process ".$mypid, LOGGER_DEBUG); - return false; - } - - // Constantly check the number of parallel database processes - if ($a->max_processes_reached()) { - logger("Max processes reached for process ".$mypid, LOGGER_DEBUG); - return false; - } - - // Constantly check the number of available database connections to let the frontend be accessible at any time - if (poller_max_connections_reached()) { - logger("Max connection reached for process ".$mypid, LOGGER_DEBUG); - return false; - } - - $argv = json_decode($queue["parameter"]); - - // Check for existance and validity of the include file - $include = $argv[0]; - - if (!validate_include($include)) { - logger("Include file ".$argv[0]." is not valid!"); - dba::delete('workerqueue', array('id' => $queue["id"])); - return true; - } - - require_once($include); - - $funcname = str_replace(".php", "", basename($argv[0]))."_run"; - - if (function_exists($funcname)) { - - // We constantly update the "executed" date every minute to avoid being killed too soon - if (!isset($poller_last_update)) { - $poller_last_update = strtotime($queue["executed"]); - } - - $age = (time() - $poller_last_update) / 60; - $poller_last_update = time(); - - if ($age > 1) { - $stamp = (float)microtime(true); - dba::update('workerqueue', array('executed' => datetime_convert()), array('pid' => $mypid, 'done' => false)); - $poller_db_duration += (microtime(true) - $stamp); - } - - poller_exec_function($queue, $funcname, $argv); - - $stamp = (float)microtime(true); - if (dba::update('workerqueue', array('done' => true), array('id' => $queue["id"]))) { - Config::set('system', 'last_poller_execution', datetime_convert()); - } - $poller_db_duration = (microtime(true) - $stamp); - } else { - logger("Function ".$funcname." does not exist"); - dba::delete('workerqueue', array('id' => $queue["id"])); - } - - return true; -} - -/** - * @brief Execute a function from the queue - * - * @param array $queue Workerqueue entry - * @param string $funcname name of the function - * @param array $argv Array of values to be passed to the function - */ -function poller_exec_function($queue, $funcname, $argv) { - global $poller_up_start, $poller_db_duration, $poller_lock_duration; - - $a = get_app(); - - $mypid = getmypid(); - - $argc = count($argv); - - $new_process_id = uniqid("wrk", true); - - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]." - Process PID: ".$new_process_id); - - $stamp = (float)microtime(true); - - // We use the callstack here to analyze the performance of executed worker entries. - // For this reason the variables have to be initialized. - if (Config::get("system", "profiler")) { - $a->performance["start"] = microtime(true); - $a->performance["database"] = 0; - $a->performance["database_write"] = 0; - $a->performance["network"] = 0; - $a->performance["file"] = 0; - $a->performance["rendering"] = 0; - $a->performance["parser"] = 0; - $a->performance["marktime"] = 0; - $a->performance["markstart"] = microtime(true); - $a->callstack = array(); - } - - // For better logging create a new process id for every worker call - // But preserve the old one for the worker - $old_process_id = $a->process_id; - $a->process_id = $new_process_id; - $a->queue = $queue; - - $up_duration = number_format(microtime(true) - $poller_up_start, 3); - - // Reset global data to avoid interferences - unset($_SESSION); - - $funcname($argv, $argc); - - $a->process_id = $old_process_id; - unset($a->queue); - - $duration = number_format(microtime(true) - $stamp, 3); - - $poller_up_start = microtime(true); - - /* With these values we can analyze how effective the worker is. - * The database and rest time should be low since this is the unproductive time. - * The execution time is the productive time. - * By changing parameters like the maximum number of workers we can check the effectivness. - */ - logger('DB: '.number_format($poller_db_duration, 2). - ' - Lock: '.number_format($poller_lock_duration, 2). - ' - Rest: '.number_format($up_duration - $poller_db_duration - $poller_lock_duration, 2). - ' - Execution: '.number_format($duration, 2), LOGGER_DEBUG); - $poller_lock_duration = 0; - - if ($duration > 3600) { - logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 1 hour (".round($duration/60, 3).")", LOGGER_DEBUG); - } elseif ($duration > 600) { - logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 10 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); - } elseif ($duration > 300) { - logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 5 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); - } elseif ($duration > 120) { - logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); - } - - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds. Process PID: ".$new_process_id); - - // Write down the performance values into the log - if (Config::get("system", "profiler")) { - $duration = microtime(true)-$a->performance["start"]; - - if (Config::get("rendertime", "callstack")) { - if (isset($a->callstack["database"])) { - $o = "\nDatabase Read:\n"; - foreach ($a->callstack["database"] AS $func => $time) { - $time = round($time, 3); - if ($time > 0) { - $o .= $func.": ".$time."\n"; - } - } - } - if (isset($a->callstack["database_write"])) { - $o .= "\nDatabase Write:\n"; - foreach ($a->callstack["database_write"] AS $func => $time) { - $time = round($time, 3); - if ($time > 0) { - $o .= $func.": ".$time."\n"; - } - } - } - if (isset($a->callstack["network"])) { - $o .= "\nNetwork:\n"; - foreach ($a->callstack["network"] AS $func => $time) { - $time = round($time, 3); - if ($time > 0) { - $o .= $func.": ".$time."\n"; - } - } - } - } else { - $o = ''; - } - - logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s".$o, - number_format($a->performance["database"] - $a->performance["database_write"], 2), - number_format($a->performance["database_write"], 2), - number_format($a->performance["network"], 2), - number_format($a->performance["file"], 2), - number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), - number_format($duration, 2)), - LOGGER_DEBUG); - } - - $cooldown = Config::get("system", "worker_cooldown", 0); - - if ($cooldown > 0) { - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); - sleep($cooldown); - } -} - -/** - * @brief Checks if the number of database connections has reached a critical limit. - * - * @return bool Are more than 3/4 of the maximum connections used? - */ -function poller_max_connections_reached() { - - // Fetch the max value from the config. This is needed when the system cannot detect the correct value by itself. - $max = Config::get("system", "max_connections"); - - // Fetch the percentage level where the poller will get active - $maxlevel = Config::get("system", "max_connections_level", 75); - - if ($max == 0) { - // the maximum number of possible user connections can be a system variable - $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_user_connections'"); - if (dbm::is_result($r)) { - $max = $r[0]["Value"]; - } - // Or it can be granted. This overrides the system variable - $r = q("SHOW GRANTS"); - if (dbm::is_result($r)) { - foreach ($r AS $grants) { - $grant = array_pop($grants); - if (stristr($grant, "GRANT USAGE ON")) { - if (preg_match("/WITH MAX_USER_CONNECTIONS (\d*)/", $grant, $match)) { - $max = $match[1]; - } - } - } - } - } - - // If $max is set we will use the processlist to determine the current number of connections - // The processlist only shows entries of the current user - if ($max != 0) { - $r = q("SHOW PROCESSLIST"); - if (!dbm::is_result($r)) { - return false; - } - $used = count($r); - - logger("Connection usage (user values): ".$used."/".$max, LOGGER_DEBUG); - - $level = ($used / $max) * 100; - - if ($level >= $maxlevel) { - logger("Maximum level (".$maxlevel."%) of user connections reached: ".$used."/".$max); - return true; - } - } - - // We will now check for the system values. - // This limit could be reached although the user limits are fine. - $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_connections'"); - if (!dbm::is_result($r)) { - return false; - } - $max = intval($r[0]["Value"]); - if ($max == 0) { - return false; - } - $r = q("SHOW STATUS WHERE `variable_name` = 'Threads_connected'"); - if (!dbm::is_result($r)) { - return false; - } - $used = intval($r[0]["Value"]); - if ($used == 0) { - return false; - } - logger("Connection usage (system values): ".$used."/".$max, LOGGER_DEBUG); - - $level = $used / $max * 100; - - if ($level < $maxlevel) { - return false; - } - logger("Maximum level (".$level."%) of system connections reached: ".$used."/".$max); - return true; -} - -/** - * @brief fix the queue entry if the worker process died - * - */ -function poller_kill_stale_workers() { - $entries = dba::select('workerqueue', array('id', 'pid', 'executed', 'priority', 'parameter'), - array('`executed` > ? AND NOT `done` AND `pid` != 0', NULL_DATE), - array('order' => array('priority', 'created'))); - while ($entry = dba::fetch($entries)) { - if (!posix_kill($entry["pid"], 0)) { - dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0), - array('id' => $entry["id"])); - } else { - // Kill long running processes - // Check if the priority is in a valid range - if (!in_array($entry["priority"], array(PRIORITY_CRITICAL, PRIORITY_HIGH, PRIORITY_MEDIUM, PRIORITY_LOW, PRIORITY_NEGLIGIBLE))) { - $entry["priority"] = PRIORITY_MEDIUM; - } - - // Define the maximum durations - $max_duration_defaults = array(PRIORITY_CRITICAL => 720, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 720); - $max_duration = $max_duration_defaults[$entry["priority"]]; - - $argv = json_decode($entry["parameter"]); - $argv[0] = basename($argv[0]); - - // How long is the process already running? - $duration = (time() - strtotime($entry["executed"])) / 60; - if ($duration > $max_duration) { - logger("Worker process ".$entry["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now."); - posix_kill($entry["pid"], SIGTERM); - - // We killed the stale process. - // To avoid a blocking situation we reschedule the process at the beginning of the queue. - // Additionally we are lowering the priority. (But not PRIORITY_CRITICAL) - if ($entry["priority"] == PRIORITY_HIGH) { - $new_priority = PRIORITY_MEDIUM; - } elseif ($entry["priority"] == PRIORITY_MEDIUM) { - $new_priority = PRIORITY_LOW; - } elseif ($entry["priority"] != PRIORITY_CRITICAL) { - $new_priority = PRIORITY_NEGLIGIBLE; - } - dba::update('workerqueue', - array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => $new_priority, 'pid' => 0), - array('id' => $entry["id"])); - } else { - logger("Worker process ".$entry["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG); - } - } - } -} - -/** - * @brief Checks if the number of active workers exceeds the given limits - * - * @return bool Are there too much workers running? - */ -function poller_too_much_workers() { - $queues = Config::get("system", "worker_queues", 4); - - $maxqueues = $queues; - - $active = poller_active_workers(); - - // Decrease the number of workers at higher load - $load = current_load(); - if ($load) { - $maxsysload = intval(Config::get("system", "maxloadavg", 50)); - - $maxworkers = $queues; - - // Some magical mathemathics to reduce the workers - $exponent = 3; - $slope = $maxworkers / pow($maxsysload, $exponent); - $queues = ceil($slope * pow(max(0, $maxsysload - $load), $exponent)); - - if (Config::get('system', 'worker_debug')) { - // Create a list of queue entries grouped by their priority - $listitem = array(); - - // Adding all processes with no workerqueue entry - $processes = dba::p("SELECT COUNT(*) AS `running` FROM `process` WHERE NOT EXISTS - (SELECT id FROM `workerqueue` - WHERE `workerqueue`.`pid` = `process`.`pid` AND NOT `done` AND `pid` != ?)", getmypid()); - if ($process = dba::fetch($processes)) { - $listitem[0] = "0:".$process["running"]; - } - dba::close($processes); - - // Now adding all processes with workerqueue entries - $entries = dba::p("SELECT COUNT(*) AS `entries`, `priority` FROM `workerqueue` WHERE NOT `done` GROUP BY `priority`"); - while ($entry = dba::fetch($entries)) { - $processes = dba::p("SELECT COUNT(*) AS `running` FROM `process` INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done` WHERE `priority` = ?", $entry["priority"]); - if ($process = dba::fetch($processes)) { - $listitem[$entry["priority"]] = $entry["priority"].":".$process["running"]."/".$entry["entries"]; - } - dba::close($processes); - } - dba::close($entries); - - $intervals = array(1, 10, 60); - $jobs_per_minute = array(); - foreach ($intervals AS $interval) { - $jobs = dba::p("SELECT COUNT(*) AS `jobs` FROM `workerqueue` WHERE `done` AND `executed` > UTC_TIMESTAMP() - INTERVAL ".intval($interval)." MINUTE"); - if ($job = dba::fetch($jobs)) { - $jobs_per_minute[$interval] = number_format($job['jobs'] / $interval, 0); - } - dba::close($jobs); - } - $processlist = ' - jpm: '.implode('/', $jobs_per_minute).' ('.implode(', ', $listitem).')'; - } - - $entries = poller_total_entries(); - - if (Config::get("system", "worker_fastlane", false) && ($queues > 0) && ($entries > 0) && ($active >= $queues)) { - $top_priority = poller_highest_priority(); - $high_running = poller_process_with_priority_active($top_priority); - - if (!$high_running && ($top_priority > PRIORITY_UNDEFINED) && ($top_priority < PRIORITY_NEGLIGIBLE)) { - logger("There are jobs with priority ".$top_priority." waiting but none is executed. Open a fastlane.", LOGGER_DEBUG); - $queues = $active + 1; - } - } - - logger("Load: ".$load."/".$maxsysload." - processes: ".$active."/".$entries.$processlist." - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG); - - // Are there fewer workers running as possible? Then fork a new one. - if (!Config::get("system", "worker_dont_fork") && ($queues > ($active + 1)) && ($entries > 1)) { - logger("Active workers: ".$active."/".$queues." Fork a new worker.", LOGGER_DEBUG); - $args = array("include/poller.php", "no_cron"); - get_app()->proc_run($args); - } - } - - return $active >= $queues; -} - -/** - * @brief Returns the number of active poller processes - * - * @return integer Number of active poller processes - */ -function poller_active_workers() { - $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'poller.php'"); - - return $workers[0]["processes"]; -} - -/** - * @brief Check if we should pass some slow processes - * - * When the active processes of the highest priority are using more than 2/3 - * of all processes, we let pass slower processes. - * - * @param string $highest_priority Returns the currently highest priority - * @return bool We let pass a slower process than $highest_priority - */ -function poller_passing_slow(&$highest_priority) { - - $highest_priority = 0; - - $r = q("SELECT `priority` - FROM `process` - INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done`"); - - // No active processes at all? Fine - if (!dbm::is_result($r)) { - return false; - } - $priorities = array(); - foreach ($r AS $line) { - $priorities[] = $line["priority"]; - } - // Should not happen - if (count($priorities) == 0) { - return false; - } - $highest_priority = min($priorities); - - // The highest process is already the slowest one? - // Then we quit - if ($highest_priority == PRIORITY_NEGLIGIBLE) { - return false; - } - $high = 0; - foreach ($priorities AS $priority) { - if ($priority == $highest_priority) { - ++$high; - } - } - logger("Highest priority: ".$highest_priority." Total processes: ".count($priorities)." Count high priority processes: ".$high, LOGGER_DEBUG); - $passing_slow = (($high/count($priorities)) > (2/3)); - - if ($passing_slow) { - logger("Passing slower processes than priority ".$highest_priority, LOGGER_DEBUG); - } - return $passing_slow; -} - -/** - * @brief Find and claim the next worker process for us - * - * @param boolean $passing_slow Returns if we had passed low priority processes - * @return boolean Have we found something? - */ -function find_worker_processes(&$passing_slow) { - - $mypid = getmypid(); - - // Check if we should pass some low priority process - $highest_priority = 0; - $found = false; - $passing_slow = false; - - // The higher the number of parallel workers, the more we prefetch to prevent concurring access - // We decrease the limit with the number of entries left in the queue - $worker_queues = Config::get("system", "worker_queues", 4); - $queue_length = Config::get('system', 'worker_fetch_limit', 1); - $lower_job_limit = $worker_queues * $queue_length * 2; - $jobs = poller_total_entries(); - - // Now do some magic - $exponent = 2; - $slope = $queue_length / pow($lower_job_limit, $exponent); - $limit = min($queue_length, ceil($slope * pow($jobs, $exponent))); - - logger('Total: '.$jobs.' - Maximum: '.$queue_length.' - jobs per queue: '.$limit, LOGGER_DEBUG); - - if (poller_passing_slow($highest_priority)) { - // Are there waiting processes with a higher priority than the currently highest? - $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND `priority` < ? AND NOT `done`", NULL_DATE, $highest_priority), - array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); - - while ($id = dba::fetch($result)) { - $ids[] = $id["id"]; - } - dba::close($result); - - $found = (count($ids) > 0); - - if (!$found) { - // Give slower processes some processing time - $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND `priority` > ? AND NOT `done`", NULL_DATE, $highest_priority), - array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); - - while ($id = dba::fetch($result)) { - $ids[] = $id["id"]; - } - dba::close($result); - - $found = (count($ids) > 0); - $passing_slow = $found; - } - } - - // If there is no result (or we shouldn't pass lower processes) we check without priority limit - if (!$found) { - $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND NOT `done`", NULL_DATE), - array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); - - while ($id = dba::fetch($result)) { - $ids[] = $id["id"]; - } - dba::close($result); - - $found = (count($ids) > 0); - } - - if ($found) { - $condition = "`id` IN (".substr(str_repeat("?, ", count($ids)), 0, -2).") AND `pid` = 0 AND NOT `done`"; - array_unshift($ids, $condition); - dba::update('workerqueue', array('executed' => datetime_convert(), 'pid' => $mypid), $ids); - } - - return $found; -} - -/** - * @brief Returns the next worker process - * - * @param boolean $passing_slow Returns if we had passed low priority processes - * @return string SQL statement - */ -function poller_worker_process(&$passing_slow) { - global $poller_db_duration, $poller_lock_duration; - - $stamp = (float)microtime(true); - - // There can already be jobs for us in the queue. - $r = q("SELECT * FROM `workerqueue` WHERE `pid` = %d AND NOT `done`", intval(getmypid())); - if (dbm::is_result($r)) { - $poller_db_duration += (microtime(true) - $stamp); - return $r; - } - - $stamp = (float)microtime(true); - if (!Lock::set('poller_worker_process')) { - return false; - } - $poller_lock_duration = (microtime(true) - $stamp); - - $stamp = (float)microtime(true); - $found = find_worker_processes($passing_slow); - $poller_db_duration += (microtime(true) - $stamp); - - Lock::remove('poller_worker_process'); - - if ($found) { - $r = q("SELECT * FROM `workerqueue` WHERE `pid` = %d AND NOT `done`", intval(getmypid())); - } - return $r; -} - -/** - * @brief Removes a workerqueue entry from the current process - */ -function poller_unclaim_process() { - $mypid = getmypid(); - - dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0), array('pid' => $mypid, 'done' => false)); -} - -/** - * @brief Call the front end worker - */ -function call_worker() { - if (!Config::get("system", "frontend_worker")) { - return; - } - - $url = System::baseUrl()."/worker"; - fetch_url($url, false, $redirects, 1); -} - -/** - * @brief Call the front end worker if there aren't any active - */ -function call_worker_if_idle() { - if (!Config::get("system", "frontend_worker")) { - return; - } - - // Do we have "proc_open"? Then we can fork the poller - if (function_exists("proc_open")) { - // When was the last time that we called the worker? - // Less than one minute? Then we quit - if ((time() - Config::get("system", "worker_started")) < 60) { - return; - } - - set_config("system", "worker_started", time()); - - // Do we have enough running workers? Then we quit here. - if (poller_too_much_workers()) { - // Cleaning dead processes - poller_kill_stale_workers(); - get_app()->remove_inactive_processes(); - - return; - } - - poller_run_cron(); - - logger('Call poller', LOGGER_DEBUG); - - $args = array("include/poller.php", "no_cron"); - get_app()->proc_run($args); - return; - } - - // We cannot execute background processes. - // We now run the processes from the frontend. - // This won't work with long running processes. - poller_run_cron(); - - clear_worker_processes(); - - $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'worker.php'"); - - if ($workers[0]["processes"] == 0) { - call_worker(); - } -} - -/** - * @brief Removes long running worker processes - */ -function clear_worker_processes() { - $timeout = Config::get("system", "frontend_worker_timeout", 10); - - /// @todo We should clean up the corresponding workerqueue entries as well - q("DELETE FROM `process` WHERE `created` < '%s' AND `command` = 'worker.php'", - dbesc(datetime_convert('UTC','UTC',"now - ".$timeout." minutes"))); -} - -/** - * @brief Runs the cron processes - */ -function poller_run_cron() { - logger('Add cron entries', LOGGER_DEBUG); - - // Check for spooled items - proc_run(PRIORITY_HIGH, "include/spool_post.php"); - - // Run the cron job that calls all other jobs - proc_run(PRIORITY_MEDIUM, "include/cron.php"); - - // Run the cronhooks job separately from cron for being able to use a different timing - proc_run(PRIORITY_MEDIUM, "include/cronhooks.php"); - - // Cleaning dead processes - poller_kill_stale_workers(); -} - -if (array_search(__file__,get_included_files())===0) { - poller_run($_SERVER["argv"],$_SERVER["argc"]); - - poller_unclaim_process(); + Worker::unclaimProcess(); get_app()->end_process(); diff --git a/index.php b/index.php index c94675a59a..fe871ba967 100644 --- a/index.php +++ b/index.php @@ -16,6 +16,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; +use Friendica\Core\Worker; require_once 'boot.php'; require_once 'object/BaseObject.php'; @@ -105,9 +106,7 @@ if (!$a->is_backend()) { session_start(); $a->save_timestamp($stamp1, "parser"); } else { - require_once "include/poller.php"; - - call_worker_if_idle(); + Worker::executeIfIdle(); } /** diff --git a/mod/worker.php b/mod/worker.php index 947656ab7c..930a807e38 100644 --- a/mod/worker.php +++ b/mod/worker.php @@ -5,6 +5,7 @@ */ require_once("include/poller.php"); +use Friendica\Core\Worker; use Friendica\Core\Config; use Friendica\Core\PConfig; @@ -17,11 +18,11 @@ function worker_init($a){ // We don't need the following lines if we can execute background jobs. // So we just wake up the worker if it sleeps. if (function_exists("proc_open")) { - call_worker_if_idle(); + Worker::executeIfIdle(); return; } - clear_worker_processes(); + Worker::clearProcesses(); $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'worker.php'"); @@ -33,22 +34,22 @@ function worker_init($a){ logger("Front end worker started: ".getmypid()); - call_worker(); + Worker::callWorker(); - if ($r = poller_worker_process()) { + if ($r = Worker::workerProcess()) { // On most configurations this parameter wouldn't have any effect. // But since it doesn't destroy anything, we just try to get more execution time in any way. set_time_limit(0); if (poller_claim_process($r[0])) { - poller_execute($r[0]); + Worker::execute($r[0]); } } - call_worker(); + Worker::callWorker(); - poller_unclaim_process(); + Worker::unclaimProcess(); $a->end_process(); diff --git a/src/Core/Worker.php b/src/Core/Worker.php new file mode 100644 index 0000000000..19a74aa4f5 --- /dev/null +++ b/src/Core/Worker.php @@ -0,0 +1,993 @@ +maxload_reached()) { + logger('Pre check: maximum load reached, quitting.', LOGGER_DEBUG); + return; + } + + // We now start the process. This is done after the load check since this could increase the load. + $a->start_process(); + + // Kill stale processes every 5 minutes + $last_cleanup = Config::get('system', 'poller_last_cleaned', 0); + if (time() > ($last_cleanup + 300)) { + Config::set('system', 'poller_last_cleaned', time()); + self::killStaleWorkers(); + } + + // Count active workers and compare them with a maximum value that depends on the load + if (self::tooMuchWorkers()) { + logger('Pre check: Active worker limit reached, quitting.', LOGGER_DEBUG); + return; + } + + // Do we have too few memory? + if ($a->min_memory_reached()) { + logger('Pre check: Memory limit reached, quitting.', LOGGER_DEBUG); + return; + } + + // Possibly there are too much database connections + if (self::maxConnectionsReached()) { + logger('Pre check: maximum connections reached, quitting.', LOGGER_DEBUG); + return; + } + + // Possibly there are too much database processes that block the system + if ($a->max_processes_reached()) { + logger('Pre check: maximum processes reached, quitting.', LOGGER_DEBUG); + return; + } + + // Now we start additional cron processes if we should do so + if ($run_cron) { + self::runCron(); + } + + $starttime = time(); + + // We fetch the next queue entry that is about to be executed + while ($r = self::workerProcess($passing_slow)) { + + // When we are processing jobs with a lower priority, we don't refetch new jobs + // Otherwise fast jobs could wait behind slow ones and could be blocked. + $refetched = $passing_slow; + + foreach ($r AS $entry) { + // Assure that the priority is an integer value + $entry['priority'] = (int)$entry['priority']; + + // The work will be done + if (!self::execute($entry)) { + logger('Process execution failed, quitting.', LOGGER_DEBUG); + return; + } + + // If possible we will fetch new jobs for this worker + if (!$refetched && Lock::set('poller_worker_process', 0)) { + $stamp = (float)microtime(true); + $refetched = self::findWorkerProcesses($passing_slow); + self::$db_duration += (microtime(true) - $stamp); + Lock::remove('poller_worker_process'); + } + } + + // To avoid the quitting of multiple pollers only one poller at a time will execute the check + if (Lock::set('poller_worker', 0)) { + $stamp = (float)microtime(true); + // Count active workers and compare them with a maximum value that depends on the load + if (self::tooMuchWorkers()) { + logger('Active worker limit reached, quitting.', LOGGER_DEBUG); + return; + } + + // Check free memory + if ($a->min_memory_reached()) { + logger('Memory limit reached, quitting.', LOGGER_DEBUG); + return; + } + Lock::remove('poller_worker'); + self::$db_duration += (microtime(true) - $stamp); + } + + // Quit the poller once every 5 minutes + if (time() > ($starttime + 300)) { + logger('Process lifetime reached, quitting.', LOGGER_DEBUG); + return; + } + } + logger("Couldn't select a workerqueue entry, quitting.", LOGGER_DEBUG); + } + + /** + * @brief Returns the number of non executed entries in the worker queue + * + * @return integer Number of non executed entries in the worker queue + */ + private static function totalEntries() { + $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` <= '%s' AND NOT `done`", dbesc(NULL_DATE)); + if (dbm::is_result($s)) { + return $s[0]["total"]; + } else { + return 0; + } + } + + /** + * @brief Returns the highest priority in the worker queue that isn't executed + * + * @return integer Number of active poller processes + */ + private static function highestPriority() { + $s = q("SELECT `priority` FROM `workerqueue` WHERE `executed` <= '%s' AND NOT `done` ORDER BY `priority` LIMIT 1", dbesc(NULL_DATE)); + if (dbm::is_result($s)) { + return $s[0]["priority"]; + } else { + return 0; + } + } + + /** + * @brief Returns if a process with the given priority is running + * + * @param integer $priority The priority that should be checked + * + * @return integer Is there a process running with that priority? + */ + private static function processWithPriorityActive($priority) { + $s = q("SELECT `id` FROM `workerqueue` WHERE `priority` <= %d AND `executed` > '%s' AND NOT `done` LIMIT 1", + intval($priority), dbesc(NULL_DATE)); + return dbm::is_result($s); + } + + /** + * @brief Execute a worker entry + * + * @param array $queue Workerqueue entry + * + * @return boolean "true" if further processing should be stopped + */ + public static function execute($queue) { + $a = get_app(); + + $mypid = getmypid(); + + // Quit when in maintenance + if (Config::get('system', 'maintenance', true)) { + logger("Maintenance mode - quit process ".$mypid, LOGGER_DEBUG); + return false; + } + + // Constantly check the number of parallel database processes + if ($a->max_processes_reached()) { + logger("Max processes reached for process ".$mypid, LOGGER_DEBUG); + return false; + } + + // Constantly check the number of available database connections to let the frontend be accessible at any time + if (self::maxConnectionsReached()) { + logger("Max connection reached for process ".$mypid, LOGGER_DEBUG); + return false; + } + + $argv = json_decode($queue["parameter"]); + + // Check for existance and validity of the include file + $include = $argv[0]; + + // The script could be provided as full path or only with the function name + if ($include == basename($include)) { + $func = "include/".$include.".php"; + } + + if (!validate_include($include)) { + logger("Include file ".$argv[0]." is not valid!"); + dba::delete('workerqueue', array('id' => $queue["id"])); + return true; + } + + require_once($include); + + $funcname = str_replace(".php", "", basename($argv[0]))."_run"; + + if (function_exists($funcname)) { + + // We constantly update the "executed" date every minute to avoid being killed too soon + if (!isset(self::$last_update)) { + self::$last_update = strtotime($queue["executed"]); + } + + $age = (time() - self::$last_update) / 60; + self::$last_update = time(); + + if ($age > 1) { + $stamp = (float)microtime(true); + dba::update('workerqueue', array('executed' => datetime_convert()), array('pid' => $mypid, 'done' => false)); + self::$db_duration += (microtime(true) - $stamp); + } + + self::execFunction($queue, $funcname, $argv); + + $stamp = (float)microtime(true); + if (dba::update('workerqueue', array('done' => true), array('id' => $queue["id"]))) { + Config::set('system', 'last_poller_execution', datetime_convert()); + } + self::$db_duration = (microtime(true) - $stamp); + } else { + logger("Function ".$funcname." does not exist"); + dba::delete('workerqueue', array('id' => $queue["id"])); + } + + return true; + } + + /** + * @brief Execute a function from the queue + * + * @param array $queue Workerqueue entry + * @param string $funcname name of the function + * @param array $argv Array of values to be passed to the function + */ + private static function execFunction($queue, $funcname, $argv) { + $a = get_app(); + + $mypid = getmypid(); + + $argc = count($argv); + + $new_process_id = uniqid("wrk", true); + + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]." - Process PID: ".$new_process_id); + + $stamp = (float)microtime(true); + + // We use the callstack here to analyze the performance of executed worker entries. + // For this reason the variables have to be initialized. + if (Config::get("system", "profiler")) { + $a->performance["start"] = microtime(true); + $a->performance["database"] = 0; + $a->performance["database_write"] = 0; + $a->performance["network"] = 0; + $a->performance["file"] = 0; + $a->performance["rendering"] = 0; + $a->performance["parser"] = 0; + $a->performance["marktime"] = 0; + $a->performance["markstart"] = microtime(true); + $a->callstack = array(); + } + + // For better logging create a new process id for every worker call + // But preserve the old one for the worker + $old_process_id = $a->process_id; + $a->process_id = $new_process_id; + $a->queue = $queue; + + $up_duration = number_format(microtime(true) - self::$up_start, 3); + + // Reset global data to avoid interferences + unset($_SESSION); + + $funcname($argv, $argc); + + $a->process_id = $old_process_id; + unset($a->queue); + + $duration = number_format(microtime(true) - $stamp, 3); + + self::$up_start = microtime(true); + + /* With these values we can analyze how effective the worker is. + * The database and rest time should be low since this is the unproductive time. + * The execution time is the productive time. + * By changing parameters like the maximum number of workers we can check the effectivness. + */ + logger('DB: '.number_format(self::$db_duration, 2). + ' - Lock: '.number_format(self::$lock_duration, 2). + ' - Rest: '.number_format($up_duration - self::$db_duration - self::$lock_duration, 2). + ' - Execution: '.number_format($duration, 2), LOGGER_DEBUG); + self::$lock_duration = 0; + + if ($duration > 3600) { + logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 1 hour (".round($duration/60, 3).")", LOGGER_DEBUG); + } elseif ($duration > 600) { + logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 10 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); + } elseif ($duration > 300) { + logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 5 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); + } elseif ($duration > 120) { + logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", LOGGER_DEBUG); + } + + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds. Process PID: ".$new_process_id); + + // Write down the performance values into the log + if (Config::get("system", "profiler")) { + $duration = microtime(true)-$a->performance["start"]; + + if (Config::get("rendertime", "callstack")) { + if (isset($a->callstack["database"])) { + $o = "\nDatabase Read:\n"; + foreach ($a->callstack["database"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) { + $o .= $func.": ".$time."\n"; + } + } + } + if (isset($a->callstack["database_write"])) { + $o .= "\nDatabase Write:\n"; + foreach ($a->callstack["database_write"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) { + $o .= $func.": ".$time."\n"; + } + } + } + if (isset($a->callstack["network"])) { + $o .= "\nNetwork:\n"; + foreach ($a->callstack["network"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) { + $o .= $func.": ".$time."\n"; + } + } + } + } else { + $o = ''; + } + + logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s".$o, + number_format($a->performance["database"] - $a->performance["database_write"], 2), + number_format($a->performance["database_write"], 2), + number_format($a->performance["network"], 2), + number_format($a->performance["file"], 2), + number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), + number_format($duration, 2)), + LOGGER_DEBUG); + } + + $cooldown = Config::get("system", "worker_cooldown", 0); + + if ($cooldown > 0) { + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); + sleep($cooldown); + } + } + + /** + * @brief Checks if the number of database connections has reached a critical limit. + * + * @return bool Are more than 3/4 of the maximum connections used? + */ + private static function maxConnectionsReached() { + + // Fetch the max value from the config. This is needed when the system cannot detect the correct value by itself. + $max = Config::get("system", "max_connections"); + + // Fetch the percentage level where the poller will get active + $maxlevel = Config::get("system", "max_connections_level", 75); + + if ($max == 0) { + // the maximum number of possible user connections can be a system variable + $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_user_connections'"); + if (dbm::is_result($r)) { + $max = $r[0]["Value"]; + } + // Or it can be granted. This overrides the system variable + $r = q("SHOW GRANTS"); + if (dbm::is_result($r)) { + foreach ($r AS $grants) { + $grant = array_pop($grants); + if (stristr($grant, "GRANT USAGE ON")) { + if (preg_match("/WITH MAX_USER_CONNECTIONS (\d*)/", $grant, $match)) { + $max = $match[1]; + } + } + } + } + } + + // If $max is set we will use the processlist to determine the current number of connections + // The processlist only shows entries of the current user + if ($max != 0) { + $r = q("SHOW PROCESSLIST"); + if (!dbm::is_result($r)) { + return false; + } + $used = count($r); + + logger("Connection usage (user values): ".$used."/".$max, LOGGER_DEBUG); + + $level = ($used / $max) * 100; + + if ($level >= $maxlevel) { + logger("Maximum level (".$maxlevel."%) of user connections reached: ".$used."/".$max); + return true; + } + } + + // We will now check for the system values. + // This limit could be reached although the user limits are fine. + $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_connections'"); + if (!dbm::is_result($r)) { + return false; + } + $max = intval($r[0]["Value"]); + if ($max == 0) { + return false; + } + $r = q("SHOW STATUS WHERE `variable_name` = 'Threads_connected'"); + if (!dbm::is_result($r)) { + return false; + } + $used = intval($r[0]["Value"]); + if ($used == 0) { + return false; + } + logger("Connection usage (system values): ".$used."/".$max, LOGGER_DEBUG); + + $level = $used / $max * 100; + + if ($level < $maxlevel) { + return false; + } + logger("Maximum level (".$level."%) of system connections reached: ".$used."/".$max); + return true; + } + + /** + * @brief fix the queue entry if the worker process died + * + */ + private static function killStaleWorkers() { + $entries = dba::select('workerqueue', array('id', 'pid', 'executed', 'priority', 'parameter'), + array('`executed` > ? AND NOT `done` AND `pid` != 0', NULL_DATE), + array('order' => array('priority', 'created'))); + while ($entry = dba::fetch($entries)) { + if (!posix_kill($entry["pid"], 0)) { + dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0), + array('id' => $entry["id"])); + } else { + // Kill long running processes + // Check if the priority is in a valid range + if (!in_array($entry["priority"], array(PRIORITY_CRITICAL, PRIORITY_HIGH, PRIORITY_MEDIUM, PRIORITY_LOW, PRIORITY_NEGLIGIBLE))) { + $entry["priority"] = PRIORITY_MEDIUM; + } + + // Define the maximum durations + $max_duration_defaults = array(PRIORITY_CRITICAL => 720, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 720); + $max_duration = $max_duration_defaults[$entry["priority"]]; + + $argv = json_decode($entry["parameter"]); + $argv[0] = basename($argv[0]); + + // How long is the process already running? + $duration = (time() - strtotime($entry["executed"])) / 60; + if ($duration > $max_duration) { + logger("Worker process ".$entry["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now."); + posix_kill($entry["pid"], SIGTERM); + + // We killed the stale process. + // To avoid a blocking situation we reschedule the process at the beginning of the queue. + // Additionally we are lowering the priority. (But not PRIORITY_CRITICAL) + if ($entry["priority"] == PRIORITY_HIGH) { + $new_priority = PRIORITY_MEDIUM; + } elseif ($entry["priority"] == PRIORITY_MEDIUM) { + $new_priority = PRIORITY_LOW; + } elseif ($entry["priority"] != PRIORITY_CRITICAL) { + $new_priority = PRIORITY_NEGLIGIBLE; + } + dba::update('workerqueue', + array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => $new_priority, 'pid' => 0), + array('id' => $entry["id"])); + } else { + logger("Worker process ".$entry["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG); + } + } + } + } + + /** + * @brief Checks if the number of active workers exceeds the given limits + * + * @return bool Are there too much workers running? + */ + public static function tooMuchWorkers() { + $queues = Config::get("system", "worker_queues", 4); + + $maxqueues = $queues; + + $active = self::activeWorkers(); + + // Decrease the number of workers at higher load + $load = current_load(); + if ($load) { + $maxsysload = intval(Config::get("system", "maxloadavg", 50)); + + $maxworkers = $queues; + + // Some magical mathemathics to reduce the workers + $exponent = 3; + $slope = $maxworkers / pow($maxsysload, $exponent); + $queues = ceil($slope * pow(max(0, $maxsysload - $load), $exponent)); + + if (Config::get('system', 'worker_debug')) { + // Create a list of queue entries grouped by their priority + $listitem = array(); + + // Adding all processes with no workerqueue entry + $processes = dba::p("SELECT COUNT(*) AS `running` FROM `process` WHERE NOT EXISTS + (SELECT id FROM `workerqueue` + WHERE `workerqueue`.`pid` = `process`.`pid` AND NOT `done` AND `pid` != ?)", getmypid()); + if ($process = dba::fetch($processes)) { + $listitem[0] = "0:".$process["running"]; + } + dba::close($processes); + + // Now adding all processes with workerqueue entries + $entries = dba::p("SELECT COUNT(*) AS `entries`, `priority` FROM `workerqueue` WHERE NOT `done` GROUP BY `priority`"); + while ($entry = dba::fetch($entries)) { + $processes = dba::p("SELECT COUNT(*) AS `running` FROM `process` INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done` WHERE `priority` = ?", $entry["priority"]); + if ($process = dba::fetch($processes)) { + $listitem[$entry["priority"]] = $entry["priority"].":".$process["running"]."/".$entry["entries"]; + } + dba::close($processes); + } + dba::close($entries); + + $intervals = array(1, 10, 60); + $jobs_per_minute = array(); + foreach ($intervals AS $interval) { + $jobs = dba::p("SELECT COUNT(*) AS `jobs` FROM `workerqueue` WHERE `done` AND `executed` > UTC_TIMESTAMP() - INTERVAL ".intval($interval)." MINUTE"); + if ($job = dba::fetch($jobs)) { + $jobs_per_minute[$interval] = number_format($job['jobs'] / $interval, 0); + } + dba::close($jobs); + } + $processlist = ' - jpm: '.implode('/', $jobs_per_minute).' ('.implode(', ', $listitem).')'; + } + + $entries = self::totalEntries(); + + if (Config::get("system", "worker_fastlane", false) && ($queues > 0) && ($entries > 0) && ($active >= $queues)) { + $top_priority = self::highestPriority(); + $high_running = self::processWithPriorityActive($top_priority); + + if (!$high_running && ($top_priority > PRIORITY_UNDEFINED) && ($top_priority < PRIORITY_NEGLIGIBLE)) { + logger("There are jobs with priority ".$top_priority." waiting but none is executed. Open a fastlane.", LOGGER_DEBUG); + $queues = $active + 1; + } + } + + logger("Load: ".$load."/".$maxsysload." - processes: ".$active."/".$entries.$processlist." - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG); + + // Are there fewer workers running as possible? Then fork a new one. + if (!Config::get("system", "worker_dont_fork") && ($queues > ($active + 1)) && ($entries > 1)) { + logger("Active workers: ".$active."/".$queues." Fork a new worker.", LOGGER_DEBUG); + $args = array("include/poller.php", "no_cron"); + get_app()->proc_run($args); + } + } + + return $active >= $queues; + } + + /** + * @brief Returns the number of active poller processes + * + * @return integer Number of active poller processes + */ + private static function activeWorkers() { + $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'poller.php'"); + + return $workers[0]["processes"]; + } + + /** + * @brief Check if we should pass some slow processes + * + * When the active processes of the highest priority are using more than 2/3 + * of all processes, we let pass slower processes. + * + * @param string $highest_priority Returns the currently highest priority + * @return bool We let pass a slower process than $highest_priority + */ + private static function passingSlow(&$highest_priority) { + + $highest_priority = 0; + + $r = q("SELECT `priority` + FROM `process` + INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done`"); + + // No active processes at all? Fine + if (!dbm::is_result($r)) { + return false; + } + $priorities = array(); + foreach ($r AS $line) { + $priorities[] = $line["priority"]; + } + // Should not happen + if (count($priorities) == 0) { + return false; + } + $highest_priority = min($priorities); + + // The highest process is already the slowest one? + // Then we quit + if ($highest_priority == PRIORITY_NEGLIGIBLE) { + return false; + } + $high = 0; + foreach ($priorities AS $priority) { + if ($priority == $highest_priority) { + ++$high; + } + } + logger("Highest priority: ".$highest_priority." Total processes: ".count($priorities)." Count high priority processes: ".$high, LOGGER_DEBUG); + $passing_slow = (($high/count($priorities)) > (2/3)); + + if ($passing_slow) { + logger("Passing slower processes than priority ".$highest_priority, LOGGER_DEBUG); + } + return $passing_slow; + } + + /** + * @brief Find and claim the next worker process for us + * + * @param boolean $passing_slow Returns if we had passed low priority processes + * @return boolean Have we found something? + */ + private static function findWorkerProcesses(&$passing_slow) { + $mypid = getmypid(); + + // Check if we should pass some low priority process + $highest_priority = 0; + $found = false; + $passing_slow = false; + + // The higher the number of parallel workers, the more we prefetch to prevent concurring access + // We decrease the limit with the number of entries left in the queue + $worker_queues = Config::get("system", "worker_queues", 4); + $queue_length = Config::get('system', 'worker_fetch_limit', 1); + $lower_job_limit = $worker_queues * $queue_length * 2; + $jobs = self::totalEntries(); + + // Now do some magic + $exponent = 2; + $slope = $queue_length / pow($lower_job_limit, $exponent); + $limit = min($queue_length, ceil($slope * pow($jobs, $exponent))); + + logger('Total: '.$jobs.' - Maximum: '.$queue_length.' - jobs per queue: '.$limit, LOGGER_DEBUG); + + if (self::passingSlow($highest_priority)) { + // Are there waiting processes with a higher priority than the currently highest? + $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND `priority` < ? AND NOT `done`", NULL_DATE, $highest_priority), + array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); + + while ($id = dba::fetch($result)) { + $ids[] = $id["id"]; + } + dba::close($result); + + $found = (count($ids) > 0); + + if (!$found) { + // Give slower processes some processing time + $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND `priority` > ? AND NOT `done`", NULL_DATE, $highest_priority), + array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); + + while ($id = dba::fetch($result)) { + $ids[] = $id["id"]; + } + dba::close($result); + + $found = (count($ids) > 0); + $passing_slow = $found; + } + } + + // If there is no result (or we shouldn't pass lower processes) we check without priority limit + if (!$found) { + $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND NOT `done`", NULL_DATE), + array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); + + while ($id = dba::fetch($result)) { + $ids[] = $id["id"]; + } + dba::close($result); + + $found = (count($ids) > 0); + } + + if ($found) { + $condition = "`id` IN (".substr(str_repeat("?, ", count($ids)), 0, -2).") AND `pid` = 0 AND NOT `done`"; + array_unshift($ids, $condition); + dba::update('workerqueue', array('executed' => datetime_convert(), 'pid' => $mypid), $ids); + } + + return $found; + } + + /** + * @brief Returns the next worker process + * + * @param boolean $passing_slow Returns if we had passed low priority processes + * @return string SQL statement + */ + public static function workerProcess(&$passing_slow) { + $stamp = (float)microtime(true); + + // There can already be jobs for us in the queue. + $r = q("SELECT * FROM `workerqueue` WHERE `pid` = %d AND NOT `done`", intval(getmypid())); + if (dbm::is_result($r)) { + self::$db_duration += (microtime(true) - $stamp); + return $r; + } + + $stamp = (float)microtime(true); + if (!Lock::set('poller_worker_process')) { + return false; + } + self::$lock_duration = (microtime(true) - $stamp); + + $stamp = (float)microtime(true); + $found = self::findWorkerProcesses($passing_slow); + self::$db_duration += (microtime(true) - $stamp); + + Lock::remove('poller_worker_process'); + + if ($found) { + $r = q("SELECT * FROM `workerqueue` WHERE `pid` = %d AND NOT `done`", intval(getmypid())); + } + return $r; + } + + /** + * @brief Removes a workerqueue entry from the current process + */ + public static function unclaimProcess() { + $mypid = getmypid(); + + dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0), array('pid' => $mypid, 'done' => false)); + } + + /** + * @brief Call the front end worker + */ + public static function callWorker() { + if (!Config::get("system", "frontend_worker")) { + return; + } + + $url = System::baseUrl()."/worker"; + fetch_url($url, false, $redirects, 1); + } + + /** + * @brief Call the front end worker if there aren't any active + */ + public static function executeIfIdle() { + if (!Config::get("system", "frontend_worker")) { + return; + } + + // Do we have "proc_open"? Then we can fork the poller + if (function_exists("proc_open")) { + // When was the last time that we called the worker? + // Less than one minute? Then we quit + if ((time() - Config::get("system", "worker_started")) < 60) { + return; + } + + set_config("system", "worker_started", time()); + + // Do we have enough running workers? Then we quit here. + if (self::tooMuchWorkers()) { + // Cleaning dead processes + self::killStaleWorkers(); + get_app()->remove_inactive_processes(); + + return; + } + + self::runCron(); + + logger('Call poller', LOGGER_DEBUG); + + $args = array("include/poller.php", "no_cron"); + get_app()->proc_run($args); + return; + } + + // We cannot execute background processes. + // We now run the processes from the frontend. + // This won't work with long running processes. + self::runCron(); + + self::clearProcesses(); + + $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'worker.php'"); + + if ($workers[0]["processes"] == 0) { + self::callWorker(); + } + } + + /** + * @brief Removes long running worker processes + */ + public static function clearProcesses() { + $timeout = Config::get("system", "frontend_worker_timeout", 10); + + /// @todo We should clean up the corresponding workerqueue entries as well + q("DELETE FROM `process` WHERE `created` < '%s' AND `command` = 'worker.php'", + dbesc(datetime_convert('UTC','UTC',"now - ".$timeout." minutes"))); + } + + /** + * @brief Runs the cron processes + */ + private static function runCron() { + logger('Add cron entries', LOGGER_DEBUG); + + // Check for spooled items + self::add(PRIORITY_HIGH, "include/spool_post.php"); + + // Run the cron job that calls all other jobs + self::add(PRIORITY_MEDIUM, "include/cron.php"); + + // Run the cronhooks job separately from cron for being able to use a different timing + self::add(PRIORITY_MEDIUM, "include/cronhooks.php"); + + // Cleaning dead processes + self::killStaleWorkers(); + } + + /** + * @brief Adds tasks to the worker queue + * + * @param (integer|array) priority or parameter array, $cmd atrings are deprecated and are ignored + * + * next args are passed as $cmd command line + * or: Worker::add(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id); + * or: Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id); + * + * @note $cmd and string args are surrounded with "" + * + * @hooks 'proc_run' + * array $arr + * + * @return boolean "false" if proc_run couldn't be executed + */ + public static function add($cmd) { + $a = get_app(); + + $proc_args = func_get_args(); + + $args = array(); + if (!count($proc_args)) { + return false; + } + + // Preserve the first parameter + // It could contain a command, the priority or an parameter array + // If we use the parameter array we have to protect it from the following function + $run_parameter = array_shift($proc_args); + + // expand any arrays + foreach ($proc_args as $arg) { + if (is_array($arg)) { + foreach ($arg as $n) { + $args[] = $n; + } + } else { + $args[] = $arg; + } + } + + // Now we add the run parameters back to the array + array_unshift($args, $run_parameter); + + $arr = array('args' => $args, 'run_cmd' => true); + + call_hooks("proc_run", $arr); + if (!$arr['run_cmd'] || !count($args)) { + return true; + } + + $priority = PRIORITY_MEDIUM; + $dont_fork = get_config("system", "worker_dont_fork"); + $created = datetime_convert(); + + if (is_int($run_parameter)) { + $priority = $run_parameter; + } elseif (is_array($run_parameter)) { + if (isset($run_parameter['priority'])) { + $priority = $run_parameter['priority']; + } + if (isset($run_parameter['created'])) { + $created = $run_parameter['created']; + } + if (isset($run_parameter['dont_fork'])) { + $dont_fork = $run_parameter['dont_fork']; + } + } + + $argv = $args; + array_shift($argv); + + $parameters = json_encode($argv); + $found = dba::exists('workerqueue', array('parameter' => $parameters, 'done' => false)); + + // Quit if there was a database error - a precaution for the update process to 3.5.3 + if (dba::errorNo() != 0) { + return false; + } + + if (!$found) { + dba::insert('workerqueue', array('parameter' => $parameters, 'created' => $created, 'priority' => $priority)); + } + + // Should we quit and wait for the poller to be called as a cronjob? + if ($dont_fork) { + return true; + } + + // If there is a lock then we don't have to check for too much worker + if (!Lock::set('poller_worker', 0)) { + return true; + } + + // If there are already enough workers running, don't fork another one + $quit = self::tooMuchWorkers(); + Lock::remove('poller_worker'); + + if ($quit) { + return true; + } + + // Now call the poller to execute the jobs that we just added to the queue + $args = array("include/poller.php", "no_cron"); + + $a->proc_run($args); + + return true; + } +} From 478e363967165980eb826e8d109dd15770e2c784 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 5 Nov 2017 12:15:53 +0000 Subject: [PATCH 2/8] proc_run was replaced --- boot.php | 35 +++++------------------------------ include/Contact.php | 9 +++++---- include/api.php | 5 +++-- include/cron.php | 33 +++++++++++++++++---------------- include/cronhooks.php | 3 ++- include/dbclean.php | 5 +++-- include/dfrn.php | 7 ++++--- include/diaspora.php | 9 +++++---- include/directory.php | 3 ++- include/discover_poco.php | 5 +++-- include/expire.php | 13 +++++++------ include/follow.php | 3 ++- include/identity.php | 3 ++- include/items.php | 12 ++++++------ include/like.php | 5 +++-- include/message.php | 3 ++- include/notifier.php | 17 +++++++++-------- include/poller.php | 9 +++++++++ include/pubsubpublish.php | 5 +++-- include/queue.php | 5 +++-- include/socgraph.php | 13 +++++++------ include/uimport.php | 3 ++- mod/admin.php | 7 ++++--- mod/contacts.php | 3 ++- mod/dfrn_confirm.php | 5 +++-- mod/dirfind.php | 3 ++- mod/events.php | 3 ++- mod/fsuggest.php | 3 ++- mod/item.php | 7 ++++--- mod/mood.php | 5 +++-- mod/photos.php | 9 +++++---- mod/poke.php | 5 +++-- mod/profile_photo.php | 7 ++++--- mod/profiles.php | 7 ++++--- mod/register.php | 3 ++- mod/regmod.php | 3 ++- mod/settings.php | 7 ++++--- mod/tagger.php | 3 ++- mod/videos.php | 3 ++- src/Core/Worker.php | 28 ++++++++-------------------- update.php | 8 +++++--- 41 files changed, 167 insertions(+), 157 deletions(-) diff --git a/boot.php b/boot.php index bf38c911cc..89ee8115ba 100644 --- a/boot.php +++ b/boot.php @@ -611,7 +611,7 @@ function check_db($via_worker) { } if ($build != DB_UPDATE_VERSION) { // When we cannot execute the database update via the worker, we will do it directly - if (!proc_run(PRIORITY_CRITICAL, 'include/dbupdate.php') && $via_worker) { + if (!Worker::add(PRIORITY_CRITICAL, 'dbupdate') && $via_worker) { update_db(get_app()); } } @@ -1031,8 +1031,10 @@ function get_max_import_size() { } /** - * @brief Wrap calls to proc_close(proc_open()) and call hook - * so plugins can take part in process :) + * @brief deprecated function to add actions to the workerqueue + * + * Please user Worker::add instead. This function here is only needed, since it is still called by the twitter addon. + * It can be safely removed after the next release. * * @param (integer|array) priority or parameter array, $cmd atrings are deprecated and are ignored * @@ -1085,7 +1087,6 @@ function proc_run($cmd) { } $priority = PRIORITY_MEDIUM; - $dont_fork = get_config("system", "worker_dont_fork"); $created = datetime_convert(); if (is_int($run_parameter)) { @@ -1097,9 +1098,6 @@ function proc_run($cmd) { if (isset($run_parameter['created'])) { $created = $run_parameter['created']; } - if (isset($run_parameter['dont_fork'])) { - $dont_fork = $run_parameter['dont_fork']; - } } $argv = $args; @@ -1117,29 +1115,6 @@ function proc_run($cmd) { dba::insert('workerqueue', array('parameter' => $parameters, 'created' => $created, 'priority' => $priority)); } - // Should we quit and wait for the poller to be called as a cronjob? - if ($dont_fork) { - return true; - } - - // If there is a lock then we don't have to check for too much worker - if (!Lock::set('poller_worker', 0)) { - return true; - } - - // If there are already enough workers running, don't fork another one - $quit = Worker::tooMuchWorkers(); - Lock::remove('poller_worker'); - - if ($quit) { - return true; - } - - // Now call the poller to execute the jobs that we just added to the queue - $args = array("include/poller.php", "no_cron"); - - $a->proc_run($args); - return true; } diff --git a/include/Contact.php b/include/Contact.php index 8be873bd30..cbc6e6211e 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; use Friendica\Network\Probe; // Included here for completeness, but this is a very dangerous operation. @@ -26,10 +27,10 @@ function user_remove($uid) { // The user and related data will be deleted in "cron_expire_and_remove_users" (cronjobs.php) q("UPDATE `user` SET `account_removed` = 1, `account_expires_on` = UTC_TIMESTAMP() WHERE `uid` = %d", intval($uid)); - proc_run(PRIORITY_HIGH, "include/notifier.php", "removeme", $uid); + Worker::add(PRIORITY_HIGH, "notifier", "removeme", $uid); // Send an update to the directory - proc_run(PRIORITY_LOW, "include/directory.php", $r['url']); + Worker::add(PRIORITY_LOW, "directory", $r['url']); if($uid == local_user()) { unset($_SESSION['authenticated']); @@ -60,7 +61,7 @@ function contact_remove($id) { dba::delete('contact', array('id' => $id)); // Delete the rest in the background - proc_run(PRIORITY_LOW, 'include/remove_contact.php', $id); + Worker::add(PRIORITY_LOW, 'remove_contact', $id); } @@ -305,7 +306,7 @@ function get_contact_details_by_url($url, $uid = -1, $default = array()) { if ((($profile["addr"] == "") || ($profile["name"] == "")) && ($profile["gid"] != 0) && in_array($profile["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS))) { - proc_run(PRIORITY_LOW, "include/update_gcontact.php", $profile["gid"]); + Worker::add(PRIORITY_LOW, "update_gcontact", $profile["gid"]); } // Show contact details of Diaspora contacts only if connected diff --git a/include/api.php b/include/api.php index d1ec25eabe..153e6c7eeb 100644 --- a/include/api.php +++ b/include/api.php @@ -9,6 +9,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; +use Friendica\Core\Worker; require_once 'include/HTTPExceptions.php'; require_once 'include/bbcode.php'; @@ -3761,10 +3762,10 @@ $called_api = null; //$user = api_get_user(get_app()); $url = System::baseUrl() . '/profile/' . get_app()->user['nickname']; if ($url && strlen(get_config('system', 'directory'))) { - proc_run(PRIORITY_LOW, "include/directory.php", $url); + Worker::add(PRIORITY_LOW, "directory", $url); } - proc_run(PRIORITY_LOW, 'include/profile_update.php', api_user()); + Worker::add(PRIORITY_LOW, 'profile_update', api_user()); // output for client if ($data) { diff --git a/include/cron.php b/include/cron.php index 8b0fab77ab..260b143c5b 100644 --- a/include/cron.php +++ b/include/cron.php @@ -1,6 +1,7 @@ $priority, 'dont_fork' => true), 'include/onepoll.php', (int)$contact['id']); + Worker::add(array('priority' => $priority, 'dont_fork' => true), 'onepoll', (int)$contact['id']); } } } diff --git a/include/cronhooks.php b/include/cronhooks.php index 4a852935a0..aa5d41afdd 100644 --- a/include/cronhooks.php +++ b/include/cronhooks.php @@ -1,6 +1,7 @@ hooks) && array_key_exists("cron", $a->hooks)) { foreach ($a->hooks["cron"] as $hook) { logger("Calling cronhooks for '" . $hook[1] . "'", LOGGER_DEBUG); - proc_run(PRIORITY_MEDIUM, "include/cronhooks.php", $hook[1]); + Worker::add(PRIORITY_MEDIUM, "cronhooks", $hook[1]); } } diff --git a/include/dbclean.php b/include/dbclean.php index bc41bead2d..1e1dd90829 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -5,6 +5,7 @@ */ use Friendica\Core\Config; +use Friendica\Core\Worker; function dbclean_run(&$argv, &$argc) { if (!Config::get('system', 'dbclean', false)) { @@ -25,7 +26,7 @@ function dbclean_run(&$argv, &$argc) { // Execute the background script for a step when it isn't finished. // Execute step 8 and 9 only when $days is defined. if (!Config::get('system', 'finished-dbclean-'.$i, false) && (($i < 8) || ($days > 0))) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', $i); + Worker::add(PRIORITY_LOW, 'dbclean', $i); } } } else { @@ -297,6 +298,6 @@ function remove_orphans($stage = 0) { // Call it again if not all entries were purged if (($stage != 0) && ($count > 0)) { - proc_run(PRIORITY_MEDIUM, 'include/dbclean.php'); + Worker::add(PRIORITY_MEDIUM, 'dbclean'); } } diff --git a/include/dfrn.php b/include/dfrn.php index 728a043ccd..ea072652f9 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -9,6 +9,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once("include/Contact.php"); require_once("include/ostatus.php"); @@ -2019,7 +2020,7 @@ class dfrn { $changed = true; if ($entrytype == DFRN_REPLY_RC) { - proc_run(PRIORITY_HIGH, "include/notifier.php","comment-import", $current["id"]); + Worker::add(PRIORITY_HIGH, "notifier","comment-import", $current["id"]); } } @@ -2652,7 +2653,7 @@ class dfrn { if ($posted_id && $parent && ($entrytype == DFRN_REPLY_RC)) { logger("Notifying followers about comment ".$posted_id, LOGGER_DEBUG); - proc_run(PRIORITY_HIGH, "include/notifier.php", "comment-import", $posted_id); + Worker::add(PRIORITY_HIGH, "notifier", "comment-import", $posted_id); } return true; @@ -2834,7 +2835,7 @@ class dfrn { if ($entrytype == DFRN_REPLY_RC) { logger("Notifying followers about deletion of post " . $item["id"], LOGGER_DEBUG); - proc_run(PRIORITY_HIGH, "include/notifier.php","drop", $item["id"]); + Worker::add(PRIORITY_HIGH, "notifier","drop", $item["id"]); } } } diff --git a/include/diaspora.php b/include/diaspora.php index 60c7909184..173e68578c 100644 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -12,6 +12,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; use Friendica\Core\PConfig; +use Friendica\Core\Worker; use Friendica\Network\Probe; require_once 'include/items.php'; @@ -1605,7 +1606,7 @@ class Diaspora { dba::insert('sign', array('iid' => $message_id, 'signed_text' => json_encode($data))); // notify others - proc_run(PRIORITY_HIGH, "include/notifier.php", "comment-import", $message_id); + Worker::add(PRIORITY_HIGH, "notifier", "comment-import", $message_id); } return true; @@ -1915,7 +1916,7 @@ class Diaspora { dba::insert('sign', array('iid' => $message_id, 'signed_text' => json_encode($data))); // notify others - proc_run(PRIORITY_HIGH, "include/notifier.php", "comment-import", $message_id); + Worker::add(PRIORITY_HIGH, "notifier", "comment-import", $message_id); } return true; @@ -2191,7 +2192,7 @@ class Diaspora { $i = item_store($arr); if ($i) - proc_run(PRIORITY_HIGH, "include/notifier.php", "activity", $i); + Worker::add(PRIORITY_HIGH, "notifier", "activity", $i); } } } @@ -2614,7 +2615,7 @@ class Diaspora { // Now check if the retraction needs to be relayed by us if ($parent["origin"]) { // notify others - proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $item["id"]); + Worker::add(PRIORITY_HIGH, "notifier", "drop", $item["id"]); } } diff --git a/include/directory.php b/include/directory.php index e507a939f8..a2ccacf125 100644 --- a/include/directory.php +++ b/include/directory.php @@ -1,6 +1,7 @@ 250) { return; @@ -177,7 +178,7 @@ function discover_users() { if ((($server_url == "") && ($user["network"] == NETWORK_FEED)) || $force_update || poco_check_server($server_url, $user["network"])) { logger('Check profile '.$user["url"]); - proc_run(PRIORITY_LOW, "include/discover_poco.php", "check_profile", $user["url"]); + Worker::add(PRIORITY_LOW, "discover_poco", "check_profile", $user["url"]); if (++$checked > 100) { return; diff --git a/include/expire.php b/include/expire.php index bb0129aad6..c65f035d21 100644 --- a/include/expire.php +++ b/include/expire.php @@ -1,6 +1,7 @@ $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), - 'include/expire.php', 'delete'); + Worker::add(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), + 'expire', 'delete'); $r = dba::p("SELECT `uid`, `username` FROM `user` WHERE `expire` != 0"); while ($row = dba::fetch($r)) { logger('Calling expiry for user '.$row['uid'].' ('.$row['username'].')', LOGGER_DEBUG); - proc_run(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), - 'include/expire.php', (int)$row['uid']); + Worker::add(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), + 'expire', (int)$row['uid']); } dba::close($r); @@ -63,8 +64,8 @@ function expire_run(&$argv, &$argc){ if (is_array($a->hooks) && array_key_exists('expire', $a->hooks)) { foreach ($a->hooks['expire'] as $hook) { logger("Calling expire hook for '" . $hook[1] . "'", LOGGER_DEBUG); - proc_run(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), - 'include/expire.php', 'hook', $hook[1]); + Worker::add(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), + 'expire', 'hook', $hook[1]); } } diff --git a/include/follow.php b/include/follow.php index 650eb7fc89..83806ade7e 100644 --- a/include/follow.php +++ b/include/follow.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; use Friendica\Network\Probe; require_once 'include/probe.php'; @@ -250,7 +251,7 @@ function new_contact($uid, $url, $interactive = false, $network = '') { // pull feed and consume it, which should subscribe to the hub. - proc_run(PRIORITY_HIGH, "include/onepoll.php", $contact_id, "force"); + Worker::add(PRIORITY_HIGH, "onepoll", $contact_id, "force"); $r = q("SELECT `contact`.*, `user`.* FROM `contact` INNER JOIN `user` ON `contact`.`uid` = `user`.`uid` WHERE `user`.`uid` = %d AND `contact`.`self` LIMIT 1", diff --git a/include/identity.php b/include/identity.php index 3686b23d2d..b1d42373b3 100644 --- a/include/identity.php +++ b/include/identity.php @@ -5,6 +5,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once 'include/ForumManager.php'; require_once 'include/bbcode.php'; @@ -888,7 +889,7 @@ function zrl_init(App $a) { return; } - proc_run(PRIORITY_LOW, 'include/gprobe.php', $tmp_str); + Worker::add(PRIORITY_LOW, 'gprobe', $tmp_str); $arr = array('zrl' => $tmp_str, 'url' => $a->cmd); call_hooks('zrl_init', $arr); } diff --git a/include/items.php b/include/items.php index 4af4c6ab37..a939515602 100644 --- a/include/items.php +++ b/include/items.php @@ -9,6 +9,7 @@ use Friendica\Core\System; use Friendica\ParseUrl; use Friendica\Util\Lock; use Friendica\Core\Config; +use Friendica\Core\Worker; require_once 'include/bbcode.php'; require_once 'include/oembed.php'; @@ -1142,7 +1143,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f check_item_notification($current_post, $uid); if ($notify) { - proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/notifier.php", $notify_type, $current_post); + Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "notifier", $notify_type, $current_post); } return $current_post; @@ -1425,7 +1426,7 @@ function tag_deliver($uid, $item_id) { ); update_thread($item_id); - proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), 'include/notifier.php', 'tgroup', $item_id); + Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), 'notifier', 'tgroup', $item_id); } @@ -2065,8 +2066,7 @@ function item_expire($uid, $days, $network = "", $force = false) { drop_item($item['id'], false); } - proc_run(array('priority' => PRIORITY_LOW, 'dont_fork' => true), "include/notifier.php", "expire", $uid); - + Worker::add(array('priority' => PRIORITY_LOW, 'dont_fork' => true), "notifier", "expire", $uid); } /// @TODO type-hint is array @@ -2088,7 +2088,7 @@ function drop_items($items) { // multiple threads may have been deleted, send an expire notification if ($uid) { - proc_run(array('priority' => PRIORITY_LOW, 'dont_fork' => true), "include/notifier.php", "expire", $uid); + Worker::add(array('priority' => PRIORITY_LOW, 'dont_fork' => true), "notifier", "expire", $uid); } } @@ -2280,7 +2280,7 @@ function drop_item($id, $interactive = true) { $drop_id = intval($item['id']); $priority = ($interactive ? PRIORITY_HIGH : PRIORITY_LOW); - proc_run(array('priority' => $priority, 'dont_fork' => true), "include/notifier.php", "drop", $drop_id); + Worker::add(array('priority' => $priority, 'dont_fork' => true), "notifier", "drop", $drop_id); if (! $interactive) { return $owner; diff --git a/include/like.php b/include/like.php index bce1c776f3..69dfa5cda1 100644 --- a/include/like.php +++ b/include/like.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once("include/diaspora.php"); @@ -166,7 +167,7 @@ function do_like($item_id, $verb) { ); $like_item_id = $like_item['id']; - proc_run(PRIORITY_HIGH, "include/notifier.php", "like", $like_item_id); + Worker::add(PRIORITY_HIGH, "notifier", "like", $like_item_id); if (!$event_verb_flag || $like_item['verb'] == $activity) { return true; @@ -253,7 +254,7 @@ EOT; call_hooks('post_local_end', $new_item); - proc_run(PRIORITY_HIGH, "include/notifier.php", "like", $new_item_id); + Worker::add(PRIORITY_HIGH, "notifier", "like", $new_item_id); return true; } diff --git a/include/message.php b/include/message.php index d6b1601110..731a56c4c2 100644 --- a/include/message.php +++ b/include/message.php @@ -4,6 +4,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; function send_message($recipient=0, $body='', $subject='', $replyto=''){ @@ -143,7 +144,7 @@ function send_message($recipient=0, $body='', $subject='', $replyto=''){ } if ($post_id) { - proc_run(PRIORITY_HIGH, "include/notifier.php", "mail", $post_id); + Worker::add(PRIORITY_HIGH, "notifier", "mail", $post_id); return intval($post_id); } else { return -3; diff --git a/include/notifier.php b/include/notifier.php index e66fa9b0b0..4c693f70e7 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\Config; +use Friendica\Core\Worker; require_once 'include/queue_fn.php'; require_once 'include/html2plain.php'; @@ -19,7 +20,7 @@ require_once 'include/salmon.php'; /* * The notifier is typically called with: * - * proc_run(PRIORITY_HIGH, "include/notifier.php", COMMAND, ITEM_ID); + * Worker::add(PRIORITY_HIGH, "notifier", COMMAND, ITEM_ID); * * where COMMAND is one of the following: * @@ -375,7 +376,7 @@ function notifier_run(&$argv, &$argc){ // a delivery fork. private groups (forum_mode == 2) do not uplink if ((intval($parent['forum_mode']) == 1) && (! $top_level) && ($cmd !== 'uplink')) { - proc_run($a->queue['priority'], 'include/notifier.php', 'uplink', $item_id); + Worker::add($a->queue['priority'], 'notifier', 'uplink', $item_id); } $conversants = array(); @@ -514,8 +515,8 @@ function notifier_run(&$argv, &$argc){ } logger("Deliver ".$target_item["guid"]." to ".$contact['url']." via network ".$contact['network'], LOGGER_DEBUG); - proc_run(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), - 'include/delivery.php', $cmd, $item_id, (int)$contact['id']); + Worker::add(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), + 'delivery', $cmd, $item_id, (int)$contact['id']); } } @@ -579,8 +580,8 @@ function notifier_run(&$argv, &$argc){ if ((! $mail) && (! $fsuggest) && (! $followup)) { logger('notifier: delivery agent: '.$rr['name'].' '.$rr['id'].' '.$rr['network'].' '.$target_item["guid"]); - proc_run(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), - 'include/delivery.php', $cmd, $item_id, (int)$rr['id']); + Worker::add(array('priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true), + 'delivery', $cmd, $item_id, (int)$rr['id']); } } } @@ -599,8 +600,8 @@ function notifier_run(&$argv, &$argc){ logger('Activating internal PuSH for item '.$item_id, LOGGER_DEBUG); // Handling the pubsubhubbub requests - proc_run(array('priority' => PRIORITY_HIGH, 'created' => $a->queue['created'], 'dont_fork' => true), - 'include/pubsubpublish.php'); + Worker::add(array('priority' => PRIORITY_HIGH, 'created' => $a->queue['created'], 'dont_fork' => true), + 'pubsubpublish'); } logger('notifier: calling hooks', LOGGER_DEBUG); diff --git a/include/poller.php b/include/poller.php index a18d10b3c6..6eb04971c3 100644 --- a/include/poller.php +++ b/include/poller.php @@ -42,6 +42,15 @@ function poller_run($argv, $argc) { load_hooks(); + // At first check the maximum load. We shouldn't continue with a high load + if ($a->maxload_reached()) { + logger('Pre check: maximum load reached, quitting.', LOGGER_DEBUG); + return; + } + + // We now start the process. This is done after the load check since this could increase the load. + $a->start_process(); + $run_cron = (($argc <= 1) || ($argv[1] != "no_cron")); Worker::processQueue($run_cron); diff --git a/include/pubsubpublish.php b/include/pubsubpublish.php index 23348d6d70..c662198abd 100644 --- a/include/pubsubpublish.php +++ b/include/pubsubpublish.php @@ -3,6 +3,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; +use Friendica\Core\Worker; require_once('include/items.php'); require_once('include/ostatus.php'); @@ -19,8 +20,8 @@ function pubsubpublish_run(&$argv, &$argc){ foreach ($r as $rr) { logger("Publish feed to ".$rr["callback_url"], LOGGER_DEBUG); - proc_run(array('priority' => PRIORITY_HIGH, 'created' => $a->queue['created'], 'dont_fork' => true), - 'include/pubsubpublish.php', (int)$rr["id"]); + Worker::add(array('priority' => PRIORITY_HIGH, 'created' => $a->queue['created'], 'dont_fork' => true), + 'pubsubpublish', (int)$rr["id"]); } } diff --git a/include/queue.php b/include/queue.php index 865d6903a2..c398300c44 100644 --- a/include/queue.php +++ b/include/queue.php @@ -1,6 +1,7 @@ PRIORITY_HIGH, 'dont_fork' => true), 'include/pubsubpublish.php'); + Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), 'pubsubpublish'); $r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue` INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id` @@ -52,7 +53,7 @@ function queue_run(&$argv, &$argc) { if (dbm::is_result($r)) { foreach ($r as $q_item) { logger('Call queue for id '.$q_item['id']); - proc_run(array('priority' => PRIORITY_LOW, 'dont_fork' => true), "include/queue.php", (int)$q_item['id']); + Worker::add(array('priority' => PRIORITY_LOW, 'dont_fork' => true), "queue", (int)$q_item['id']); } } return; diff --git a/include/socgraph.php b/include/socgraph.php index f0a855d70c..2b9e51e4f6 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -10,6 +10,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; +use Friendica\Core\Worker; use Friendica\Network\Probe; require_once 'include/datetime.php'; @@ -39,7 +40,7 @@ require_once 'include/Photo.php'; */ function poco_load($cid, $uid = 0, $zcid = 0, $url = null) { // Call the function "poco_load_worker" via the worker - proc_run(PRIORITY_LOW, "include/discover_poco.php", "poco_load", (int)$cid, (int)$uid, (int)$zcid, $url); + Worker::add(PRIORITY_LOW, "discover_poco", "poco_load", (int)$cid, (int)$uid, (int)$zcid, $url); } /** @@ -1702,7 +1703,7 @@ function poco_fetch_serverlist($poco) { $r = q("SELECT `nurl` FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); if (!dbm::is_result($r)) { logger("Call server check for server ".$server_url, LOGGER_DEBUG); - proc_run(PRIORITY_LOW, "include/discover_poco.php", "server", $server_url); + Worker::add(PRIORITY_LOW, "discover_poco", "server", $server_url); } } } @@ -1724,7 +1725,7 @@ function poco_discover_federation() { $servers = json_decode($serverdata); foreach ($servers->pods as $server) { - proc_run(PRIORITY_LOW, "include/discover_poco.php", "server", "https://".$server->host); + Worker::add(PRIORITY_LOW, "discover_poco", "server", "https://".$server->host); } } @@ -1737,7 +1738,7 @@ function poco_discover_federation() { foreach ($servers as $server) { $url = (is_null($server->https_score) ? 'http' : 'https').'://'.$server->name; - proc_run(PRIORITY_LOW, "include/discover_poco.php", "server", $url); + Worker::add(PRIORITY_LOW, "discover_poco", "server", $url); } } } @@ -1847,7 +1848,7 @@ function poco_discover($complete = false) { } logger('Update directory from server '.$server['url'].' with ID '.$server['id'], LOGGER_DEBUG); - proc_run(PRIORITY_LOW, "include/discover_poco.php", "update_server_directory", (int)$server['id']); + Worker::add(PRIORITY_LOW, "discover_poco", "update_server_directory", (int)$server['id']); if (!$complete && (--$no_of_queries == 0)) { break; @@ -2125,7 +2126,7 @@ function get_gcontact_id($contact) { if ($doprobing) { logger("Last Contact: ". $last_contact_str." - Last Failure: ".$last_failure_str." - Checking: ".$contact["url"], LOGGER_DEBUG); - proc_run(PRIORITY_LOW, 'include/gprobe.php', $contact["url"]); + Worker::add(PRIORITY_LOW, 'gprobe', $contact["url"]); } return $gcontact_id; diff --git a/include/uimport.php b/include/uimport.php index 442df94460..fbdbeb58c6 100644 --- a/include/uimport.php +++ b/include/uimport.php @@ -3,6 +3,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\PConfig; +use Friendica\Core\Worker; require_once("include/Photo.php"); define("IMPORT_DEBUG", False); @@ -284,7 +285,7 @@ function import_account(App $a, $file) { } // send relocate messages - proc_run(PRIORITY_HIGH, 'include/notifier.php', 'relocate', $newuid); + Worker::add(PRIORITY_HIGH, 'notifier', 'relocate', $newuid); info(t("Done. You can now login with your username and password")); goaway(System::baseUrl() . "/login"); diff --git a/mod/admin.php b/mod/admin.php index 4f0622532f..144539dcef 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -9,6 +9,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; +use Friendica\Core\Worker; require_once("include/enotify.php"); require_once("include/text.php"); @@ -694,7 +695,7 @@ function admin_page_site_post(App $a) { check_form_security_token_redirectOnErr('/admin/site', 'admin_site'); if (!empty($_POST['republish_directory'])) { - proc_run(PRIORITY_LOW, 'include/directory.php'); + Worker::add(PRIORITY_LOW, 'directory'); return; } @@ -767,7 +768,7 @@ function admin_page_site_post(App $a) { $users = q("SELECT `uid` FROM `user` WHERE `account_removed` = 0 AND `account_expired` = 0"); foreach ($users as $user) { - proc_run(PRIORITY_HIGH, 'include/notifier.php', 'relocate', $user['uid']); + Worker::add(PRIORITY_HIGH, 'notifier', 'relocate', $user['uid']); } info("Relocation started. Could take a while to complete."); @@ -855,7 +856,7 @@ function admin_page_site_post(App $a) { // Has the directory url changed? If yes, then resubmit the existing profiles there if ($global_directory != Config::get('system', 'directory') && ($global_directory != '')) { Config::set('system', 'directory', $global_directory); - proc_run(PRIORITY_LOW, 'include/directory.php'); + Worker::add(PRIORITY_LOW, 'directory'); } if ($a->get_path() != "") { diff --git a/mod/contacts.php b/mod/contacts.php index 93013972f1..216d2451ea 100644 --- a/mod/contacts.php +++ b/mod/contacts.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; use Friendica\Network\Probe; require_once 'include/Contact.php'; @@ -251,7 +252,7 @@ function _contact_update($contact_id) { intval($contact_id)); } else // pull feed and consume it, which should subscribe to the hub. - proc_run(PRIORITY_HIGH, "include/onepoll.php", $contact_id, "force"); + Worker::add(PRIORITY_HIGH, "onepoll", $contact_id, "force"); } function _contact_update_profile($contact_id) { diff --git a/mod/dfrn_confirm.php b/mod/dfrn_confirm.php index 0e99b26c32..e7869e1d22 100644 --- a/mod/dfrn_confirm.php +++ b/mod/dfrn_confirm.php @@ -20,6 +20,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; use Friendica\Network\Probe; require_once 'include/enotify.php'; @@ -494,7 +495,7 @@ function dfrn_confirm_post(App $a, $handsfree = null) { $i = item_store($arr); if($i) - proc_run(PRIORITY_HIGH, "include/notifier.php", "activity", $i); + Worker::add(PRIORITY_HIGH, "notifier", "activity", $i); } } } @@ -796,7 +797,7 @@ function dfrn_confirm_post(App $a, $handsfree = null) { $i = item_store($arr); if($i) - proc_run(PRIORITY_HIGH, "include/notifier.php", "activity", $i); + Worker::add(PRIORITY_HIGH, "notifier", "activity", $i); } } diff --git a/mod/dirfind.php b/mod/dirfind.php index 7d167494e6..a1253b1442 100644 --- a/mod/dirfind.php +++ b/mod/dirfind.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once 'include/contact_widgets.php'; require_once 'include/probe.php'; @@ -164,7 +165,7 @@ function dirfind_content(App $a, $prefix = "") { } // Add found profiles from the global directory to the local directory - proc_run(PRIORITY_LOW, 'include/discover_poco.php', "dirsearch", urlencode($search)); + Worker::add(PRIORITY_LOW, 'discover_poco', "dirsearch", urlencode($search)); } else { $p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : ''); diff --git a/mod/events.php b/mod/events.php index de65790a77..c89684f74c 100644 --- a/mod/events.php +++ b/mod/events.php @@ -6,6 +6,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once 'include/bbcode.php'; require_once 'include/datetime.php'; @@ -177,7 +178,7 @@ function events_post(App $a) { $item_id = event_store($datarray); if (! $cid) { - proc_run(PRIORITY_HIGH, "include/notifier.php", "event", $item_id); + Worker::add(PRIORITY_HIGH, "notifier", "event", $item_id); } goaway($_SESSION['return_url']); diff --git a/mod/fsuggest.php b/mod/fsuggest.php index 32ed63b4c2..efaf100efc 100644 --- a/mod/fsuggest.php +++ b/mod/fsuggest.php @@ -1,6 +1,7 @@ PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id); + Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "create_shadowentry", $post_id); // Call the background process that is delivering the item to the receivers - proc_run(PRIORITY_HIGH, "include/notifier.php", $notify_type, $post_id); + Worker::add(PRIORITY_HIGH, "notifier", $notify_type, $post_id); logger('post_complete'); diff --git a/mod/mood.php b/mod/mood.php index c4d93b6de2..26eb736d19 100644 --- a/mod/mood.php +++ b/mod/mood.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once('include/security.php'); require_once('include/bbcode.php'); @@ -98,13 +99,13 @@ function mood_init(App $a) { intval($uid), intval($item_id) ); - proc_run(PRIORITY_HIGH, "include/notifier.php", "tag", $item_id); + Worker::add(PRIORITY_HIGH, "notifier", "tag", $item_id); } call_hooks('post_local_end', $arr); - proc_run(PRIORITY_HIGH, "include/notifier.php", "like", $post_id); + Worker::add(PRIORITY_HIGH, "notifier", "like", $post_id); return; } diff --git a/mod/photos.php b/mod/photos.php index bafd97ebee..e626ef4005 100644 --- a/mod/photos.php +++ b/mod/photos.php @@ -3,6 +3,7 @@ use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; +use Friendica\Core\Worker; use Friendica\Network\Probe; require_once 'include/Photo.php'; @@ -304,7 +305,7 @@ function photos_post(App $a) { // send the notification upstream/downstream as the case may be if ($rr['visible']) { - proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id); + Worker::add(PRIORITY_HIGH, "notifier", "drop", $drop_id); } } } @@ -381,7 +382,7 @@ function photos_post(App $a) { photo_albums($page_owner_uid, true); if ($i[0]['visible']) { - proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id); + Worker::add(PRIORITY_HIGH, "notifier", "drop", $drop_id); } } } @@ -729,7 +730,7 @@ function photos_post(App $a) { $item_id = item_store($arr); if ($item_id) { - proc_run(PRIORITY_HIGH, "include/notifier.php", "tag", $item_id); + Worker::add(PRIORITY_HIGH, "notifier", "tag", $item_id); } } } @@ -934,7 +935,7 @@ function photos_post(App $a) { photo_albums($page_owner_uid, true); if ($visible) { - proc_run(PRIORITY_HIGH, "include/notifier.php", 'wall-new', $item_id); + Worker::add(PRIORITY_HIGH, "notifier", 'wall-new', $item_id); } call_hooks('photo_post_end',intval($item_id)); diff --git a/mod/poke.php b/mod/poke.php index ca02b6bee5..c02d59c460 100644 --- a/mod/poke.php +++ b/mod/poke.php @@ -16,6 +16,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once('include/security.php'); require_once('include/bbcode.php'); @@ -137,13 +138,13 @@ function poke_init(App $a) { // intval($uid), // intval($item_id) //); - proc_run(PRIORITY_HIGH, "include/notifier.php", "tag", $item_id); + Worker::add(PRIORITY_HIGH, "notifier", "tag", $item_id); } call_hooks('post_local_end', $arr); - proc_run(PRIORITY_HIGH, "include/notifier.php", "like", $post_id); + Worker::add(PRIORITY_HIGH, "notifier", "like", $post_id); return; } diff --git a/mod/profile_photo.php b/mod/profile_photo.php index b4723b215f..bc93144539 100644 --- a/mod/profile_photo.php +++ b/mod/profile_photo.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once("include/Photo.php"); @@ -129,10 +130,10 @@ function profile_photo_post(App $a) { // Update global directory in background $url = System::baseUrl() . '/profile/' . $a->user['nickname']; if ($url && strlen(get_config('system','directory'))) { - proc_run(PRIORITY_LOW, "include/directory.php", $url); + Worker::add(PRIORITY_LOW, "directory", $url); } - proc_run(PRIORITY_LOW, 'include/profile_update.php', local_user()); + Worker::add(PRIORITY_LOW, 'profile_update', local_user()); } else { notice( t('Unable to process image') . EOL); } @@ -229,7 +230,7 @@ function profile_photo_content(App $a) { // Update global directory in background $url = $_SESSION['my_url']; if ($url && strlen(get_config('system','directory'))) { - proc_run(PRIORITY_LOW, "include/directory.php", $url); + Worker::add(PRIORITY_LOW, "directory", $url); } goaway(System::baseUrl() . '/profiles'); diff --git a/mod/profiles.php b/mod/profiles.php index 4a1d1ad934..90296322a2 100644 --- a/mod/profiles.php +++ b/mod/profiles.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; use Friendica\Network\Probe; require_once 'include/Contact.php'; @@ -497,10 +498,10 @@ function profiles_post(App $a) { // Update global directory in background $url = $_SESSION['my_url']; if ($url && strlen(get_config('system', 'directory'))) { - proc_run(PRIORITY_LOW, "include/directory.php", $url); + Worker::add(PRIORITY_LOW, "directory", $url); } - proc_run(PRIORITY_LOW, 'include/profile_update.php', local_user()); + Worker::add(PRIORITY_LOW, 'profile_update', local_user()); // Update the global contact for the user update_gcontact_for_user(local_user()); @@ -594,7 +595,7 @@ function profile_activity($changed, $value) { $i = item_store($arr); if ($i) { - proc_run(PRIORITY_HIGH, "include/notifier.php", "activity", $i); + Worker::add(PRIORITY_HIGH, "notifier", "activity", $i); } } diff --git a/mod/register.php b/mod/register.php index cb9c1729f5..d3a7e00924 100644 --- a/mod/register.php +++ b/mod/register.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once('include/enotify.php'); require_once('include/bbcode.php'); @@ -69,7 +70,7 @@ function register_post(App $a) { if($netpublish && $a->config['register_policy'] != REGISTER_APPROVE) { $url = System::baseUrl() . '/profile/' . $user['nickname']; - proc_run(PRIORITY_LOW, "include/directory.php", $url); + Worker::add(PRIORITY_LOW, "directory", $url); } $using_invites = get_config('system','invitation_only'); diff --git a/mod/regmod.php b/mod/regmod.php index a2c454f9a2..be57c8d656 100644 --- a/mod/regmod.php +++ b/mod/regmod.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once('include/enotify.php'); require_once('include/user.php'); @@ -42,7 +43,7 @@ function user_allow($hash) { if (dbm::is_result($r) && $r[0]['net-publish']) { $url = System::baseUrl() . '/profile/' . $user[0]['nickname']; if ($url && strlen(get_config('system','directory'))) { - proc_run(PRIORITY_LOW, "include/directory.php", $url); + Worker::add(PRIORITY_LOW, "directory", $url); } } diff --git a/mod/settings.php b/mod/settings.php index e8d24f43de..b4d8d338f9 100644 --- a/mod/settings.php +++ b/mod/settings.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once('include/group.php'); require_once('include/socgraph.php'); @@ -355,7 +356,7 @@ function settings_post(App $a) { check_form_security_token_redirectOnErr('/settings', 'settings'); if (x($_POST,'resend_relocate')) { - proc_run(PRIORITY_HIGH, 'include/notifier.php', 'relocate', local_user()); + Worker::add(PRIORITY_HIGH, 'notifier', 'relocate', local_user()); info(t("Relocate message has been send to your contacts")); goaway('settings'); } @@ -629,11 +630,11 @@ function settings_post(App $a) { // Update global directory in background $url = $_SESSION['my_url']; if ($url && strlen(get_config('system','directory'))) { - proc_run(PRIORITY_LOW, "include/directory.php", $url); + Worker::add(PRIORITY_LOW, "directory", $url); } } - proc_run(PRIORITY_LOW, 'include/profile_update.php', local_user()); + Worker::add(PRIORITY_LOW, 'profile_update', local_user()); // Update the global contact for the user update_gcontact_for_user(local_user()); diff --git a/mod/tagger.php b/mod/tagger.php index ea15b3f403..84e5e5615c 100644 --- a/mod/tagger.php +++ b/mod/tagger.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once('include/security.php'); require_once('include/bbcode.php'); @@ -214,7 +215,7 @@ EOT; call_hooks('post_local_end', $arr); - proc_run(PRIORITY_HIGH, "include/notifier.php", "tag", $post_id); + Worker::add(PRIORITY_HIGH, "notifier", "tag", $post_id); killme(); diff --git a/mod/videos.php b/mod/videos.php index 09b44151d8..c75e8b943f 100644 --- a/mod/videos.php +++ b/mod/videos.php @@ -2,6 +2,7 @@ use Friendica\App; use Friendica\Core\System; +use Friendica\Core\Worker; require_once('include/items.php'); require_once('include/acl_selectors.php'); @@ -170,7 +171,7 @@ function videos_post(App $a) { $drop_id = intval($i[0]['id']); if ($i[0]['visible']) { - proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id); + Worker::add(PRIORITY_HIGH, "notifier", "drop", $drop_id); } } } diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 19a74aa4f5..5964f44046 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -10,7 +10,7 @@ use dba; use dbm; /** - * @file include/Core/Worker.php + * @file src/Core/Worker.php * * @brief Contains the class for all worker relevant stuff */ @@ -34,15 +34,6 @@ class Worker { self::$up_start = microtime(true); - // At first check the maximum load. We shouldn't continue with a high load - if ($a->maxload_reached()) { - logger('Pre check: maximum load reached, quitting.', LOGGER_DEBUG); - return; - } - - // We now start the process. This is done after the load check since this could increase the load. - $a->start_process(); - // Kill stale processes every 5 minutes $last_cleanup = Config::get('system', 'poller_last_cleaned', 0); if (time() > ($last_cleanup + 300)) { @@ -212,7 +203,7 @@ class Worker { // The script could be provided as full path or only with the function name if ($include == basename($include)) { - $func = "include/".$include.".php"; + $include = "include/".$include.".php"; } if (!validate_include($include)) { @@ -868,13 +859,13 @@ class Worker { logger('Add cron entries', LOGGER_DEBUG); // Check for spooled items - self::add(PRIORITY_HIGH, "include/spool_post.php"); + self::add(PRIORITY_HIGH, "spool_post"); // Run the cron job that calls all other jobs - self::add(PRIORITY_MEDIUM, "include/cron.php"); + self::add(PRIORITY_MEDIUM, "cron"); // Run the cronhooks job separately from cron for being able to use a different timing - self::add(PRIORITY_MEDIUM, "include/cronhooks.php"); + self::add(PRIORITY_MEDIUM, "cronhooks"); // Cleaning dead processes self::killStaleWorkers(); @@ -886,8 +877,8 @@ class Worker { * @param (integer|array) priority or parameter array, $cmd atrings are deprecated and are ignored * * next args are passed as $cmd command line - * or: Worker::add(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id); - * or: Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id); + * or: Worker::add(PRIORITY_HIGH, "notifier", "drop", $drop_id); + * or: Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "create_shadowentry", $post_id); * * @note $cmd and string args are surrounded with "" * @@ -897,8 +888,6 @@ class Worker { * @return boolean "false" if proc_run couldn't be executed */ public static function add($cmd) { - $a = get_app(); - $proc_args = func_get_args(); $args = array(); @@ -985,8 +974,7 @@ class Worker { // Now call the poller to execute the jobs that we just added to the queue $args = array("include/poller.php", "no_cron"); - - $a->proc_run($args); + get_app()->proc_run($args); return true; } diff --git a/update.php b/update.php index 413317f73b..db67d54600 100644 --- a/update.php +++ b/update.php @@ -2,6 +2,8 @@ define('UPDATE_VERSION' , 1235); +use Friendica\Core\Worker; + /** * * update.php - automatic system update @@ -1596,7 +1598,7 @@ function update_1169() { if (!$r) return UPDATE_FAILED; - proc_run(PRIORITY_LOW, "include/threadupdate.php"); + Worker::add(PRIORITY_LOW, "threadupdate"); return UPDATE_SUCCESS; } @@ -1637,7 +1639,7 @@ function update_1178() { set_config('system','community_page_style', CP_NO_COMMUNITY_PAGE); // Update the central item storage with uid=0 - proc_run(PRIORITY_LOW, "include/threadupdate.php"); + Worker::add(PRIORITY_LOW, "threadupdate"); return UPDATE_SUCCESS; } @@ -1645,7 +1647,7 @@ function update_1178() { function update_1180() { // Fill the new fields in the term table. - proc_run(PRIORITY_LOW, "include/tagupdate.php"); + Worker::add(PRIORITY_LOW, "tagupdate"); return UPDATE_SUCCESS; } From 9039e60a06e8762c87290c5c2826a3661046ed2b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 5 Nov 2017 12:32:35 +0000 Subject: [PATCH 3/8] proc_run is not needed anymore --- boot.php | 88 -------------------------------------------------------- 1 file changed, 88 deletions(-) diff --git a/boot.php b/boot.php index 89ee8115ba..d8659805dc 100644 --- a/boot.php +++ b/boot.php @@ -1030,94 +1030,6 @@ function get_max_import_size() { return ((x($a->config, 'max_import_size')) ? $a->config['max_import_size'] : 0 ); } -/** - * @brief deprecated function to add actions to the workerqueue - * - * Please user Worker::add instead. This function here is only needed, since it is still called by the twitter addon. - * It can be safely removed after the next release. - * - * @param (integer|array) priority or parameter array, $cmd atrings are deprecated and are ignored - * - * next args are passed as $cmd command line - * or: proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id); - * or: proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id); - * - * @note $cmd and string args are surrounded with "" - * - * @hooks 'proc_run' - * array $arr - * - * @return boolean "false" if proc_run couldn't be executed - */ -function proc_run($cmd) { - - $a = get_app(); - - $proc_args = func_get_args(); - - $args = array(); - if (!count($proc_args)) { - return false; - } - - // Preserve the first parameter - // It could contain a command, the priority or an parameter array - // If we use the parameter array we have to protect it from the following function - $run_parameter = array_shift($proc_args); - - // expand any arrays - foreach ($proc_args as $arg) { - if (is_array($arg)) { - foreach ($arg as $n) { - $args[] = $n; - } - } else { - $args[] = $arg; - } - } - - // Now we add the run parameters back to the array - array_unshift($args, $run_parameter); - - $arr = array('args' => $args, 'run_cmd' => true); - - call_hooks("proc_run", $arr); - if (!$arr['run_cmd'] || ! count($args)) { - return true; - } - - $priority = PRIORITY_MEDIUM; - $created = datetime_convert(); - - if (is_int($run_parameter)) { - $priority = $run_parameter; - } elseif (is_array($run_parameter)) { - if (isset($run_parameter['priority'])) { - $priority = $run_parameter['priority']; - } - if (isset($run_parameter['created'])) { - $created = $run_parameter['created']; - } - } - - $argv = $args; - array_shift($argv); - - $parameters = json_encode($argv); - $found = dba::exists('workerqueue', array('parameter' => $parameters, 'done' => false)); - - // Quit if there was a database error - a precaution for the update process to 3.5.3 - if (dba::errorNo() != 0) { - return false; - } - - if (!$found) { - dba::insert('workerqueue', array('parameter' => $parameters, 'created' => $created, 'priority' => $priority)); - } - - return true; -} - function current_theme() { $app_base_themes = array('duepuntozero', 'dispy', 'quattro'); From 70c08dee1dfa6960a611961e8ff23fffb7e15177 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 5 Nov 2017 15:28:55 +0000 Subject: [PATCH 4/8] Some more improvements --- boot.php | 13 ++++++++++++- include/poller.php | 10 ---------- src/Core/Worker.php | 27 +++++++++++++++++++-------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/boot.php b/boot.php index d8659805dc..35c37d715e 100644 --- a/boot.php +++ b/boot.php @@ -22,7 +22,6 @@ require_once(__DIR__ . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'a use Friendica\App; use Friendica\Core\System; -use Friendica\Core\Worker; use Friendica\Core\Config; use Friendica\Util\Lock; @@ -1030,6 +1029,18 @@ function get_max_import_size() { return ((x($a->config, 'max_import_size')) ? $a->config['max_import_size'] : 0 ); } +/** + * @brief compatibilty wrapper for Worker::add function + * + * @param (integer|array) priority or parameter array, $cmd atrings are deprecated and are ignored + * + * @return boolean "false" if proc_run couldn't be executed + */ +function proc_run() { + $proc_args = func_get_args(); + call_user_func_array('Friendica\Core\Worker::add', $proc_args); +} + function current_theme() { $app_base_themes = array('duepuntozero', 'dispy', 'quattro'); diff --git a/include/poller.php b/include/poller.php index 6eb04971c3..3f6290a98f 100644 --- a/include/poller.php +++ b/include/poller.php @@ -42,17 +42,7 @@ function poller_run($argv, $argc) { load_hooks(); - // At first check the maximum load. We shouldn't continue with a high load - if ($a->maxload_reached()) { - logger('Pre check: maximum load reached, quitting.', LOGGER_DEBUG); - return; - } - - // We now start the process. This is done after the load check since this could increase the load. - $a->start_process(); - $run_cron = (($argc <= 1) || ($argv[1] != "no_cron")); - Worker::processQueue($run_cron); return; } diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 5964f44046..ac5c04a271 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -4,6 +4,7 @@ namespace Friendica\Core; use Friendica\App; use Friendica\Core\System; use Friendica\Core\Config; +use Friendica\Core\Worker; use Friendica\Util\Lock; use dba; @@ -34,6 +35,15 @@ class Worker { self::$up_start = microtime(true); + // At first check the maximum load. We shouldn't continue with a high load + if ($a->maxload_reached()) { + logger('Pre check: maximum load reached, quitting.', LOGGER_DEBUG); + return; + } + + // We now start the process. This is done after the load check since this could increase the load. + $a->start_process(); + // Kill stale processes every 5 minutes $last_cleanup = Config::get('system', 'poller_last_cleaned', 0); if (time() > ($last_cleanup + 300)) { @@ -589,8 +599,7 @@ class Worker { // Are there fewer workers running as possible? Then fork a new one. if (!Config::get("system", "worker_dont_fork") && ($queues > ($active + 1)) && ($entries > 1)) { logger("Active workers: ".$active."/".$queues." Fork a new worker.", LOGGER_DEBUG); - $args = array("include/poller.php", "no_cron"); - get_app()->proc_run($args); + self::spawnWorker(); } } @@ -603,7 +612,7 @@ class Worker { * @return integer Number of active poller processes */ private static function activeWorkers() { - $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'poller.php'"); + $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'Worker.php'"); return $workers[0]["processes"]; } @@ -821,9 +830,7 @@ class Worker { self::runCron(); logger('Call poller', LOGGER_DEBUG); - - $args = array("include/poller.php", "no_cron"); - get_app()->proc_run($args); + self::spawnWorker(); return; } @@ -871,6 +878,11 @@ class Worker { self::killStaleWorkers(); } + public static function spawnWorker() { + $args = array("include/poller.php", "no_cron"); + get_app()->proc_run($args); + } + /** * @brief Adds tasks to the worker queue * @@ -973,8 +985,7 @@ class Worker { } // Now call the poller to execute the jobs that we just added to the queue - $args = array("include/poller.php", "no_cron"); - get_app()->proc_run($args); + self::spawnWorker(); return true; } From 84c3c0753e10f267c6fa5b81a7130b0af167da9b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 5 Nov 2017 17:13:37 +0000 Subject: [PATCH 5/8] Now we use only dba functions --- src/Core/Worker.php | 83 +++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index ac5c04a271..31b17d36ef 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -141,9 +141,9 @@ class Worker { * @return integer Number of non executed entries in the worker queue */ private static function totalEntries() { - $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` <= '%s' AND NOT `done`", dbesc(NULL_DATE)); + $s = dba::fetch_first("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` <= ? AND NOT `done`", NULL_DATE); if (dbm::is_result($s)) { - return $s[0]["total"]; + return $s["total"]; } else { return 0; } @@ -155,9 +155,10 @@ class Worker { * @return integer Number of active poller processes */ private static function highestPriority() { - $s = q("SELECT `priority` FROM `workerqueue` WHERE `executed` <= '%s' AND NOT `done` ORDER BY `priority` LIMIT 1", dbesc(NULL_DATE)); + $condition = array("`executed` <= ? AND NOT `done`", NULL_DATE); + $s = dba::select('workerqueue', array('priority'), $condition, array('limit' => 1, 'order' => array('priority'))); if (dbm::is_result($s)) { - return $s[0]["priority"]; + return $s["priority"]; } else { return 0; } @@ -171,9 +172,8 @@ class Worker { * @return integer Is there a process running with that priority? */ private static function processWithPriorityActive($priority) { - $s = q("SELECT `id` FROM `workerqueue` WHERE `priority` <= %d AND `executed` > '%s' AND NOT `done` LIMIT 1", - intval($priority), dbesc(NULL_DATE)); - return dbm::is_result($s); + $condition = array("`priority` <= ? AND `executed` > ? AND NOT `done`", $priority, NULL_DATE); + return dba::exists('workerqueue', $condition); } /** @@ -404,32 +404,29 @@ class Worker { if ($max == 0) { // the maximum number of possible user connections can be a system variable - $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_user_connections'"); + $r = dba::fetch_first("SHOW VARIABLES WHERE `variable_name` = 'max_user_connections'"); if (dbm::is_result($r)) { - $max = $r[0]["Value"]; + $max = $r["Value"]; } // Or it can be granted. This overrides the system variable - $r = q("SHOW GRANTS"); - if (dbm::is_result($r)) { - foreach ($r AS $grants) { - $grant = array_pop($grants); - if (stristr($grant, "GRANT USAGE ON")) { - if (preg_match("/WITH MAX_USER_CONNECTIONS (\d*)/", $grant, $match)) { - $max = $match[1]; - } + $r = dba::p('SHOW GRANTS'); + while ($grants = dba::fetch($r)) { + $grant = array_pop($grants); + if (stristr($grant, "GRANT USAGE ON")) { + if (preg_match("/WITH MAX_USER_CONNECTIONS (\d*)/", $grant, $match)) { + $max = $match[1]; } } } + dba::close($r); } // If $max is set we will use the processlist to determine the current number of connections // The processlist only shows entries of the current user if ($max != 0) { - $r = q("SHOW PROCESSLIST"); - if (!dbm::is_result($r)) { - return false; - } - $used = count($r); + $r = dba::p('SHOW PROCESSLIST'); + $used = dba::num_rows($r); + dba::close($r); logger("Connection usage (user values): ".$used."/".$max, LOGGER_DEBUG); @@ -443,19 +440,19 @@ class Worker { // We will now check for the system values. // This limit could be reached although the user limits are fine. - $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_connections'"); + $r = dba::fetch_first("SHOW VARIABLES WHERE `variable_name` = 'max_connections'"); if (!dbm::is_result($r)) { return false; } - $max = intval($r[0]["Value"]); + $max = intval($r["Value"]); if ($max == 0) { return false; } - $r = q("SHOW STATUS WHERE `variable_name` = 'Threads_connected'"); + $r = dba::fetch_first("SHOW STATUS WHERE `variable_name` = 'Threads_connected'"); if (!dbm::is_result($r)) { return false; } - $used = intval($r[0]["Value"]); + $used = intval($r["Value"]); if ($used == 0) { return false; } @@ -612,9 +609,9 @@ class Worker { * @return integer Number of active poller processes */ private static function activeWorkers() { - $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'Worker.php'"); + $workers = dba::fetch_first("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'Worker.php'"); - return $workers[0]["processes"]; + return $workers["processes"]; } /** @@ -627,21 +624,22 @@ class Worker { * @return bool We let pass a slower process than $highest_priority */ private static function passingSlow(&$highest_priority) { - $highest_priority = 0; - $r = q("SELECT `priority` - FROM `process` - INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done`"); + $r = dba::p("SELECT `priority` + FROM `process` + INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done`"); // No active processes at all? Fine if (!dbm::is_result($r)) { return false; } $priorities = array(); - foreach ($r AS $line) { + while ($line = dba::fetch($r)) { $priorities[] = $line["priority"]; } + dba::close($r); + // Should not happen if (count($priorities) == 0) { return false; @@ -755,11 +753,12 @@ class Worker { $stamp = (float)microtime(true); // There can already be jobs for us in the queue. - $r = q("SELECT * FROM `workerqueue` WHERE `pid` = %d AND NOT `done`", intval(getmypid())); + $r = dba::select('workerqueue', array(), array('pid' => getmypid(), 'done' => false)); if (dbm::is_result($r)) { self::$db_duration += (microtime(true) - $stamp); - return $r; + return dba::inArray($r); } + dba::close($r); $stamp = (float)microtime(true); if (!Lock::set('poller_worker_process')) { @@ -774,9 +773,10 @@ class Worker { Lock::remove('poller_worker_process'); if ($found) { - $r = q("SELECT * FROM `workerqueue` WHERE `pid` = %d AND NOT `done`", intval(getmypid())); + $r = dba::select('workerqueue', array(), array('pid' => getmypid(), 'done' => false)); + return dba::inArray($r); } - return $r; + return false; } /** @@ -841,9 +841,9 @@ class Worker { self::clearProcesses(); - $workers = q("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'worker.php'"); + $workers = dba::fetch_first("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'worker.php'"); - if ($workers[0]["processes"] == 0) { + if ($workers["processes"] == 0) { self::callWorker(); } } @@ -855,8 +855,9 @@ class Worker { $timeout = Config::get("system", "frontend_worker_timeout", 10); /// @todo We should clean up the corresponding workerqueue entries as well - q("DELETE FROM `process` WHERE `created` < '%s' AND `command` = 'worker.php'", - dbesc(datetime_convert('UTC','UTC',"now - ".$timeout." minutes"))); + $condition = array("`created` < ? AND `command` = 'worker.php'", + datetime_convert('UTC','UTC',"now - ".$timeout." minutes")); + dba::delete('process', $condition); } /** From 4ebb76e5a6124707d7bd00aa23d7adfc8f301027 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 6 Nov 2017 15:32:34 +0000 Subject: [PATCH 6/8] Changed explanation --- src/Core/Worker.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 31b17d36ef..5cedff8236 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -13,7 +13,7 @@ use dbm; /** * @file src/Core/Worker.php * - * @brief Contains the class for all worker relevant stuff + * @brief Contains the class for the worker background job processing */ /** From 28701a8df3bf2d0435be77e934ccc1a9295ecbf6 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 6 Nov 2017 15:38:15 +0000 Subject: [PATCH 7/8] Some small correction --- src/Core/Worker.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 5cedff8236..bf0767c07c 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -887,7 +887,7 @@ class Worker { /** * @brief Adds tasks to the worker queue * - * @param (integer|array) priority or parameter array, $cmd atrings are deprecated and are ignored + * @param (integer|array) priority or parameter array, strings are deprecated and are ignored * * next args are passed as $cmd command line * or: Worker::add(PRIORITY_HIGH, "notifier", "drop", $drop_id); From 1607527447b7bda29a278e23e1c04371058f2b61 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 6 Nov 2017 15:40:27 +0000 Subject: [PATCH 8/8] And some more --- boot.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boot.php b/boot.php index 35c37d715e..2adafac60f 100644 --- a/boot.php +++ b/boot.php @@ -1032,7 +1032,7 @@ function get_max_import_size() { /** * @brief compatibilty wrapper for Worker::add function * - * @param (integer|array) priority or parameter array, $cmd atrings are deprecated and are ignored + * @param (integer|array) priority or parameter array, strings are deprecated and are ignored * * @return boolean "false" if proc_run couldn't be executed */