Browse Source

Create Core\Process as a base for followup work

tags/2019.09
Philipp Holzer 6 months ago
parent
commit
2c73377b39
No account linked to committer's email address
4 changed files with 227 additions and 145 deletions
  1. +15
    -144
      src/App.php
  2. +203
    -0
      src/Core/Process.php
  3. +4
    -1
      src/Core/Worker.php
  4. +5
    -0
      static/dependencies.config.php

+ 15
- 144
src/App.php View File

@@ -127,6 +127,11 @@ class App
*/
private $args;

/**
* @var Core\Process The process methods
*/
private $process;

/**
* Returns the current config cache of this node
*
@@ -225,8 +230,9 @@ class App
* @param Profiler $profiler The profiler of this application
* @param L10n $l10n The translator instance
* @param App\Arguments $args The Friendica Arguments of the call
* @param Core\Process $process The process methods
*/
public function __construct(Database $database, Configuration $config, App\Mode $mode, BaseURL $baseURL, LoggerInterface $logger, Profiler $profiler, L10n $l10n, Arguments $args, App\Module $module, App\Page $page)
public function __construct(Database $database, Configuration $config, App\Mode $mode, BaseURL $baseURL, LoggerInterface $logger, Profiler $profiler, L10n $l10n, Arguments $args, App\Module $module, App\Page $page, Core\Process $process)
{
$this->database = $database;
$this->config = $config;
@@ -236,6 +242,7 @@ class App
$this->logger = $logger;
$this->l10n = $l10n;
$this->args = $args;
$this->process = $process;

$this->cmd = $args->getCommand();
$this->argv = $args->getArgv();
@@ -398,163 +405,27 @@ class App
}

/**
* @brief Checks if the maximum number of database processes is reached
*
* @return bool Is the limit reached?
* @deprecated 2019.09 - use Core\Process->isMaxProcessesReached() instead
*/
public function isMaxProcessesReached()
{
// Deactivated, needs more investigating if this check really makes sense
return false;

/*
* Commented out to suppress static analyzer issues
*
if ($this->is_backend()) {
$process = 'backend';
$max_processes = $this->config->get('system', 'max_processes_backend');
if (intval($max_processes) == 0) {
$max_processes = 5;
}
} else {
$process = 'frontend';
$max_processes = $this->config->get('system', 'max_processes_frontend');
if (intval($max_processes) == 0) {
$max_processes = 20;
}
}

$processlist = DBA::processlist();
if ($processlist['list'] != '') {
$this->logger->debug('Processcheck: Processes: ' . $processlist['amount'] . ' - Processlist: ' . $processlist['list']);

if ($processlist['amount'] > $max_processes) {
$this->logger->debug('Processcheck: Maximum number of processes for ' . $process . ' tasks (' . $max_processes . ') reached.');
return true;
}
}
return false;
*/
return $this->process->isMaxProcessesReached();
}

/**
* @brief Checks if the minimal memory is reached
*
* @return bool Is the memory limit reached?
* @throws HTTPException\InternalServerErrorException
* @deprecated 2019.09 - use Core\Process->isMinMemoryReached() instead
*/
public function isMinMemoryReached()
{
$min_memory = $this->config->get('system', 'min_memory', 0);
if ($min_memory == 0) {
return false;
}

if (!is_readable('/proc/meminfo')) {
return false;
}

$memdata = explode("\n", file_get_contents('/proc/meminfo'));

$meminfo = [];
foreach ($memdata as $line) {
$data = explode(':', $line);
if (count($data) != 2) {
continue;
}
list($key, $val) = $data;
$meminfo[$key] = (int)trim(str_replace('kB', '', $val));
$meminfo[$key] = (int)($meminfo[$key] / 1024);
}

if (!isset($meminfo['MemFree'])) {
return false;
}

$free = $meminfo['MemFree'];

$reached = ($free < $min_memory);

if ($reached) {
$this->logger->debug('Minimal memory reached.', ['free' => $free, 'memtotal' => $meminfo['MemTotal'], 'limit' => $min_memory]);
}

return $reached;
return $this->process->isMinMemoryReached();
}

/**
* @brief Checks if the maximum load is reached
*
* @return bool Is the load reached?
* @throws HTTPException\InternalServerErrorException
* @deprecated 2019.09 - use Core\Process->isMaxLoadReached() instead
*/
public function isMaxLoadReached()
{
if ($this->mode->isBackend()) {
$process = 'backend';
$maxsysload = intval($this->config->get('system', 'maxloadavg'));
if ($maxsysload < 1) {
$maxsysload = 50;
}
} else {
$process = 'frontend';
$maxsysload = intval($this->config->get('system', 'maxloadavg_frontend'));
if ($maxsysload < 1) {
$maxsysload = 50;
}
}

$load = Core\System::currentLoad();
if ($load) {
if (intval($load) > $maxsysload) {
$this->logger->info('system load for process too high.', ['load' => $load, 'process' => $process, 'maxsysload' => $maxsysload]);
return true;
}
}
return false;
}

/**
* Executes a child process with 'proc_open'
*
* @param string $command The command to execute
* @param array $args Arguments to pass to the command ( [ 'key' => value, 'key2' => value2, ... ]
*
* @throws HTTPException\InternalServerErrorException
*/
public function proc_run($command, $args)
{
if (!function_exists('proc_open')) {
return;
}

$cmdline = $this->config->get('config', 'php_path', 'php') . ' ' . escapeshellarg($command);

foreach ($args as $key => $value) {
if (!is_null($value) && is_bool($value) && !$value) {
continue;
}

$cmdline .= ' --' . $key;
if (!is_null($value) && !is_bool($value)) {
$cmdline .= ' ' . $value;
}
}

if ($this->isMinMemoryReached()) {
return;
}

if (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') {
$resource = proc_open('cmd /c start /b ' . $cmdline, [], $foo, $this->getBasePath());
} else {
$resource = proc_open($cmdline . ' &', [], $foo, $this->getBasePath());
}
if (!is_resource($resource)) {
$this->logger->debug('We got no resource for command.', ['cmd' => $cmdline]);
return;
}
proc_close($resource);
return $this->process->isMaxLoadReached();
}

/**
@@ -729,7 +600,7 @@ class App
}

// Max Load Average reached: ERROR
if ($this->isMaxProcessesReached() || $this->isMaxLoadReached()) {
if ($this->process->isMaxProcessesReached() || $this->process->isMaxLoadReached()) {
header('Retry-After: 120');
header('Refresh: 120; url=' . $this->baseURL->get() . "/" . $this->args->getQueryString());


+ 203
- 0
src/Core/Process.php View File

@@ -0,0 +1,203 @@
<?php

namespace Friendica\Core;

use Friendica\App;
use Friendica\Core\Config\Configuration;
use Psr\Log\LoggerInterface;

/**
* Methods for interacting with the current process or create new process
*
* @todo 2019.12 Next release, this class holds all process relevant methods based on the big Worker class
* - Starting new processes (including checks)
* - Enabling multi-node processing (e.g. for docker service)
* - Using an process-id per node
* - Using memory locks for multi-node locking (redis, memcached, ..)
*/
final class Process
{
/**
* @var LoggerInterface
*/
private $logger;

/**
* @var App\Mode
*/
private $mode;

/**
* @var Configuration
*/
private $config;

/**
* @var string
*/
private $basePath;

public function __construct(LoggerInterface $logger, App\Mode $mode, Configuration $config, string $basepath)
{
$this->logger = $logger;
$this->mode = $mode;
$this->config = $config;
$this->basePath = $basepath;
}

/**
* @brief Checks if the maximum number of database processes is reached
*
* @return bool Is the limit reached?
*/
public function isMaxProcessesReached()
{
// Deactivated, needs more investigating if this check really makes sense
return false;

/*
* Commented out to suppress static analyzer issues
*
if ($this->mode->isBackend()) {
$process = 'backend';
$max_processes = $this->config->get('system', 'max_processes_backend');
if (intval($max_processes) == 0) {
$max_processes = 5;
}
} else {
$process = 'frontend';
$max_processes = $this->config->get('system', 'max_processes_frontend');
if (intval($max_processes) == 0) {
$max_processes = 20;
}
}

$processlist = DBA::processlist();
if ($processlist['list'] != '') {
$this->logger->debug('Processcheck: Processes: ' . $processlist['amount'] . ' - Processlist: ' . $processlist['list']);

if ($processlist['amount'] > $max_processes) {
$this->logger->debug('Processcheck: Maximum number of processes for ' . $process . ' tasks (' . $max_processes . ') reached.');
return true;
}
}
return false;
*/
}

/**
* @brief Checks if the minimal memory is reached
*
* @return bool Is the memory limit reached?
*/
public function isMinMemoryReached()
{
$min_memory = $this->config->get('system', 'min_memory', 0);
if ($min_memory == 0) {
return false;
}

if (!is_readable('/proc/meminfo')) {
return false;
}

$memdata = explode("\n", file_get_contents('/proc/meminfo'));

$meminfo = [];
foreach ($memdata as $line) {
$data = explode(':', $line);
if (count($data) != 2) {
continue;
}
list($key, $val) = $data;
$meminfo[$key] = (int)trim(str_replace('kB', '', $val));
$meminfo[$key] = (int)($meminfo[$key] / 1024);
}

if (!isset($meminfo['MemFree'])) {
return false;
}

$free = $meminfo['MemFree'];

$reached = ($free < $min_memory);

if ($reached) {
$this->logger->debug('Minimal memory reached.', ['free' => $free, 'memtotal' => $meminfo['MemTotal'], 'limit' => $min_memory]);
}

return $reached;
}

/**
* @brief Checks if the maximum load is reached
*
* @return bool Is the load reached?
*/
public function isMaxLoadReached()
{
if ($this->mode->isBackend()) {
$process = 'backend';
$maxsysload = intval($this->config->get('system', 'maxloadavg'));
if ($maxsysload < 1) {
$maxsysload = 50;
}
} else {
$process = 'frontend';
$maxsysload = intval($this->config->get('system', 'maxloadavg_frontend'));
if ($maxsysload < 1) {
$maxsysload = 50;
}
}

$load = System::currentLoad();
if ($load) {
if (intval($load) > $maxsysload) {
$this->logger->info('system load for process too high.', ['load' => $load, 'process' => $process, 'maxsysload' => $maxsysload]);
return true;
}
}
return false;
}

/**
* Executes a child process with 'proc_open'
*
* @param string $command The command to execute
* @param array $args Arguments to pass to the command ( [ 'key' => value, 'key2' => value2, ... ]
*/
public function run($command, $args)
{
if (!function_exists('proc_open')) {
return;
}

$cmdline = $this->config->get('config', 'php_path', 'php') . ' ' . escapeshellarg($command);

foreach ($args as $key => $value) {
if (!is_null($value) && is_bool($value) && !$value) {
continue;
}

$cmdline .= ' --' . $key;
if (!is_null($value) && !is_bool($value)) {
$cmdline .= ' ' . $value;
}
}

if ($this->isMinMemoryReached()) {
return;
}

if (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') {
$resource = proc_open('cmd /c start /b ' . $cmdline, [], $foo, $this->basePath);
} else {
$resource = proc_open($cmdline . ' &', [], $foo, $this->basePath);
}
if (!is_resource($resource)) {
$this->logger->debug('We got no resource for command.', ['cmd' => $cmdline]);
return;
}
proc_close($resource);
}
}

+ 4
- 1
src/Core/Worker.php View File

@@ -5,6 +5,7 @@
namespace Friendica\Core;

use Friendica\BaseObject;
use Friendica\Core;
use Friendica\Database\DBA;
use Friendica\Model\Process;
use Friendica\Util\DateTimeFormat;
@@ -1082,7 +1083,9 @@ class Worker

$args = ['no_cron' => !$do_cron];

get_app()->proc_run($command, $args);
$a = get_app();
$process = new Core\Process($a->getLogger(), $a->getMode(), $a->getConfig(), $a->getBasePath());
$process->run($command, $args);

// after spawning we have to remove the flag.
if (Config::get('system', 'worker_daemon_mode', false)) {

+ 5
- 0
static/dependencies.config.php View File

@@ -154,4 +154,9 @@ return [
['determineModule', [], Dice::CHAIN_CALL],
],
],
Friendica\Core\Process::class => [
'constructParams' => [
[Dice::INSTANCE => '$basepath'],
],
],
];

Loading…
Cancel
Save