From a81ac835a1215417bc920da400c89ed2c5531006 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 1 Jan 2021 23:05:26 +0000 Subject: [PATCH 1/2] The frontend worker is removed --- doc/Settings.md | 6 -- doc/de/Settings.md | 6 -- src/App.php | 5 -- src/Core/Installer.php | 7 ++ src/Core/Worker.php | 91 +----------------------- src/Module/Admin/Site.php | 18 ----- src/Module/Worker.php | 87 ---------------------- static/defaults.config.php | 4 -- view/templates/admin/site.tpl | 2 - view/theme/frio/templates/admin/site.tpl | 2 - 10 files changed, 8 insertions(+), 220 deletions(-) delete mode 100644 src/Module/Worker.php diff --git a/doc/Settings.md b/doc/Settings.md index 67a5af832..16b7ec38a 100644 --- a/doc/Settings.md +++ b/doc/Settings.md @@ -240,15 +240,9 @@ This section allows you to configure the background process that is triggered by The process does check the available system resources before creating a new worker for a task. Because of this, it may happen that the maximum number of worker processes you allow will not be reached. -If your server setup does not allow you to use the `proc_open` function of PHP, please disable it in this section. - The tasks for the background process have priorities. To guarantee that important tasks are executed even though the system has a lot of work to do, it is useful to enable the *fastlane*. -Should you not be able to run a cron job on your server, you can also activate the *frontend* worker. -If you have done so, you can call `example.com/worker` (replace example.com with your actual domain name) on a regular basis from an external service. -This will then trigger the execution of the background process. - ### Relocate ## Users diff --git a/doc/de/Settings.md b/doc/de/Settings.md index 33023fd71..34b349e88 100644 --- a/doc/de/Settings.md +++ b/doc/de/Settings.md @@ -227,15 +227,9 @@ In diesem Abschnitt kann der Hintergrund-Prozess konfiguriert werden. Bevor ein neuer *Worker* Prozess gestartet wird, überprüft das System, dass die vorhandenen Resourchen ausrechend sind, Aus diesem Grund kann es sein, dass die maximale Zahl der Hintergrungprozesse nicht erreicht wird. -Sollte die PHP Funktion `proc_open` auf dem Server nicht verfügbar sein, kann die Verwendung durch Friendica hier unterbunden werden. - Die Aufgaben die im Hintergrund erledigt werden, haben Prioritäten zugeteilt. Um garantieren zu können, das wichtige Prozesse schnellst möglich abgearbeitet werden können, selbst wenn das System gerade stark belastet ist, sollte die *fastlane* aktiviert sein. -Wenn es auf deinem Server nicht möglich ist, einen cron Job zu starten, kannst du den *frontend* Worker einschalten. -Nachdem dies geschehen ist, kannst du `example.com/worker` (tausche example.com mit dem echten Domainnamen aus) aufrufen werden. -Dadurch werden dann die Aufgaben aktiviert, die der cron Job sonst aktivieren würde. - ### Umsiedeln ## Nutzer diff --git a/src/App.php b/src/App.php index de8311848..4a9552aaf 100644 --- a/src/App.php +++ b/src/App.php @@ -445,11 +445,6 @@ class App Core\Hook::callAll('init_1'); } - // Exclude the backend processes from the session management - if ($this->mode->isBackend()) { - Core\Worker::executeIfIdle(); - } - if ($this->mode->isNormal() && !$this->mode->isBackend()) { $requester = HTTPSignature::getSigner('', $_SERVER); if (!empty($requester)) { diff --git a/src/Core/Installer.php b/src/Core/Installer.php index 4f2f2d99f..70ee4bba4 100644 --- a/src/Core/Installer.php +++ b/src/Core/Installer.php @@ -463,6 +463,13 @@ class Installer ); $returnVal = $returnVal ? $status : false; + $status = $this->checkFunction('proc_open', + DI::l10n()->t('Program execution functions'), + DI::l10n()->t('Error: Program execution functions required but not enabled.'), + true + ); + $returnVal = $returnVal ? $status : false; + $status = $this->checkFunction('json_encode', DI::l10n()->t('JSON PHP module'), DI::l10n()->t('Error: JSON PHP module required but not installed.'), diff --git a/src/Core/Worker.php b/src/Core/Worker.php index e90747e41..a0e444642 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -1072,95 +1072,6 @@ class Worker self::$db_duration_write += (microtime(true) - $stamp); } - /** - * Call the front end worker - * - * @return void - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function callWorker() - { - if (!DI::config()->get("system", "frontend_worker")) { - return; - } - - $url = DI::baseUrl() . '/worker'; - DI::httpRequest()->fetch($url, 1); - } - - /** - * Call the front end worker if there aren't any active - * - * @return void - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function executeIfIdle() - { - self::checkDaemonState(); - - if (!DI::config()->get("system", "frontend_worker")) { - return; - } - - // Do we have "proc_open"? Then we can fork the worker - if (function_exists("proc_open")) { - // When was the last time that we called the worker? - // Less than one minute? Then we quit - if ((time() - DI::config()->get("system", "worker_started")) < 60) { - return; - } - - DI::config()->set("system", "worker_started", time()); - - // Do we have enough running workers? Then we quit here. - if (self::tooMuchWorkers()) { - // Cleaning dead processes - self::killStaleWorkers(); - DI::modelProcess()->deleteInactive(); - - return; - } - - self::runCron(); - - Logger::info('Call worker'); - self::spawnWorker(); - return; - } - - // We cannot execute background processes. - // We now run the processes from the frontend. - // This won't work with long running processes. - self::runCron(); - - self::clearProcesses(); - - $workers = self::activeWorkers(); - - if ($workers == 0) { - self::callWorker(); - } - } - - /** - * Removes long running worker processes - * - * @return void - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function clearProcesses() - { - $timeout = DI::config()->get("system", "frontend_worker_timeout", 10); - - /// @todo We should clean up the corresponding workerqueue entries as well - $stamp = (float)microtime(true); - $condition = ["`created` < ? AND `command` = 'worker.php'", - DateTimeFormat::utc("now - ".$timeout." minutes")]; - DBA::delete('process', $condition); - self::$db_duration = (microtime(true) - $stamp); - self::$db_duration_write += (microtime(true) - $stamp); - } - /** * Runs the cron processes * @@ -1230,7 +1141,7 @@ class Worker { // Worker and daemon are started from the command line. // This means that this is executed by a PHP interpreter without runtime limitations - if (in_array(DI::mode()->getExecutor(), [Mode::DAEMON, Mode::WORKER])) { + if (function_exists('pcntl_fork') && in_array(DI::mode()->getExecutor(), [Mode::DAEMON, Mode::WORKER])) { self::forkProcess($do_cron); } else { $process = new Core\Process(DI::logger(), DI::mode(), DI::config(), diff --git a/src/Module/Admin/Site.php b/src/Module/Admin/Site.php index 47c9e0ae5..0e0753a40 100644 --- a/src/Module/Admin/Site.php +++ b/src/Module/Admin/Site.php @@ -205,9 +205,7 @@ class Site extends BaseAdmin $check_new_version_url = (!empty($_POST['check_new_version_url']) ? Strings::escapeTags(trim($_POST['check_new_version_url'])) : 'none'); $worker_queues = (!empty($_POST['worker_queues']) ? intval($_POST['worker_queues']) : 10); - $worker_dont_fork = !empty($_POST['worker_dont_fork']); $worker_fastlane = !empty($_POST['worker_fastlane']); - $worker_frontend = !empty($_POST['worker_frontend']); $relay_directly = !empty($_POST['relay_directly']); $relay_server = (!empty($_POST['relay_server']) ? Strings::escapeTags(trim($_POST['relay_server'])) : ''); @@ -417,13 +415,7 @@ class Site extends BaseAdmin DI::config()->set('system', 'only_tag_search' , $only_tag_search); DI::config()->set('system', 'worker_queues' , $worker_queues); - - if (function_exists('proc_open')) { - DI::config()->set('system', 'worker_dont_fork', $worker_dont_fork); - } - DI::config()->set('system', 'worker_fastlane' , $worker_fastlane); - DI::config()->set('system', 'frontend_worker' , $worker_frontend); DI::config()->set('system', 'relay_directly' , $relay_directly); DI::config()->set('system', 'relay_server' , $relay_server); @@ -582,14 +574,6 @@ class Site extends BaseAdmin } } - if (function_exists('proc_open')) { - $worker_dont_fork = DI::config()->get('system', 'worker_dont_fork'); - $worker_dont_fork_disabled = ''; - } else { - $worker_dont_fork = true; - $worker_dont_fork_disabled = 'disabled'; - } - $t = Renderer::getMarkupTemplate('admin/site.tpl'); return Renderer::replaceMacros($t, [ '$title' => DI::l10n()->t('Administration'), @@ -702,9 +686,7 @@ class Site extends BaseAdmin '$rino' => ['rino', DI::l10n()->t('RINO Encryption'), intval(DI::config()->get('system', 'rino_encrypt')), DI::l10n()->t('Encryption layer between nodes.'), [0 => DI::l10n()->t('Disabled'), 1 => DI::l10n()->t('Enabled')]], '$worker_queues' => ['worker_queues', DI::l10n()->t('Maximum number of parallel workers'), DI::config()->get('system', 'worker_queues'), DI::l10n()->t('On shared hosters set this to %d. On larger systems, values of %d are great. Default value is %d.', 5, 20, 10)], - '$worker_dont_fork' => ['worker_dont_fork', DI::l10n()->t('Don\'t use "proc_open" with the worker'), $worker_dont_fork, DI::l10n()->t('Enable this if your system doesn\'t allow the use of "proc_open". This can happen on shared hosters. If this is enabled you should increase the frequency of worker calls in your crontab.'), $worker_dont_fork_disabled], '$worker_fastlane' => ['worker_fastlane', DI::l10n()->t('Enable fastlane'), DI::config()->get('system', 'worker_fastlane'), DI::l10n()->t('When enabed, the fastlane mechanism starts an additional worker if processes with higher priority are blocked by processes of lower priority.')], - '$worker_frontend' => ['worker_frontend', DI::l10n()->t('Enable frontend worker'), DI::config()->get('system', 'frontend_worker'), DI::l10n()->t('When enabled the Worker process is triggered when backend access is performed (e.g. messages being delivered). On smaller sites you might want to call %s/worker on a regular basis via an external cron job. You should only enable this option if you cannot utilize cron/scheduled jobs on your server.', DI::baseUrl()->get())], '$relay_subscribe' => ['relay_subscribe', DI::l10n()->t('Use relay servers'), DI::config()->get('system', 'relay_subscribe'), DI::l10n()->t('Enables the receiving of public posts from relay servers. They will be included in the search, subscribed tags and on the global community page.')], '$relay_server' => ['relay_server', DI::l10n()->t('"Social Relay" server'), DI::config()->get('system', 'relay_server'), DI::l10n()->t('Address of the "Social Relay" server where public posts should be send to. For example %s. ActivityRelay servers are administrated via the "console relay" command line command.', 'https://social-relay.isurf.ca')], diff --git a/src/Module/Worker.php b/src/Module/Worker.php deleted file mode 100644 index d73ed1d8e..000000000 --- a/src/Module/Worker.php +++ /dev/null @@ -1,87 +0,0 @@ -. - * - */ - -namespace Friendica\Module; - -use Friendica\BaseModule; -use Friendica\Core\Process; -use Friendica\Core\System; -use Friendica\Core\Worker as WorkerCore; -use Friendica\Database\DBA; -use Friendica\DI; -use Friendica\Util\DateTimeFormat; - -/** - * Module for starting the backend worker through a frontend call - */ -class Worker extends BaseModule -{ - public static function rawContent(array $parameters = []) - { - if (!DI::config()->get("system", "frontend_worker")) { - return; - } - - // Ensure that all "strtotime" operations do run timezone independent - date_default_timezone_set('UTC'); - - // We don't need the following lines if we can execute background jobs. - // So we just wake up the worker if it sleeps. - if (function_exists("proc_open")) { - WorkerCore::executeIfIdle(); - return; - } - - WorkerCore::clearProcesses(); - - $workers = DBA::count('process', ['command' => 'worker.php']); - - if ($workers > DI::config()->get("system", "worker_queues", 4)) { - return; - } - - DI::process()->start(); - - DI::logger()->notice('Front end worker started.', ['pid' => getmypid()]); - - WorkerCore::callWorker(); - - if ($r = WorkerCore::workerProcess()) { - // On most configurations this parameter wouldn't have any effect. - // But since it doesn't destroy anything, we just try to get more execution time in any way. - set_time_limit(0); - - $fields = ['executed' => DateTimeFormat::utcNow(), 'pid' => getmypid(), 'done' => false]; - $condition = ['id' => $r[0]["id"], 'pid' => 0]; - if (DBA::update('workerqueue', $fields, $condition)) { - WorkerCore::execute($r[0]); - } - } - - WorkerCore::callWorker(); - - WorkerCore::unclaimProcess(); - - DI::process()->end(); - - System::httpExit(200, 'Frontend worker stopped.'); - } -} diff --git a/static/defaults.config.php b/static/defaults.config.php index 310d1ea08..c3102e388 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -243,10 +243,6 @@ return [ // Number of "free" searches when system => permit_crawling is enabled. 'free_crawls' => 10, - // frontend_worker_timeout (Integer) - // Value in minutes after we think that a frontend task was killed by the webserver. - 'frontend_worker_timeout' => 10, - // groupedit_image_limit (Integer) // Number of contacts at which the group editor should switch from display the profile pictures of the contacts to only display the names. // This can alternatively be set on a per account basis in the pconfig table. diff --git a/view/templates/admin/site.tpl b/view/templates/admin/site.tpl index 2d820b9be..c44553d6d 100644 --- a/view/templates/admin/site.tpl +++ b/view/templates/admin/site.tpl @@ -122,9 +122,7 @@ {{include file="field_input.tpl" field=$maxloadavg}} {{include file="field_input.tpl" field=$min_memory}} {{include file="field_input.tpl" field=$worker_queues}} - {{include file="field_checkbox.tpl" field=$worker_dont_fork}} {{include file="field_checkbox.tpl" field=$worker_fastlane}} - {{include file="field_checkbox.tpl" field=$worker_frontend}}
diff --git a/view/theme/frio/templates/admin/site.tpl b/view/theme/frio/templates/admin/site.tpl index 289988dd0..57f43cefb 100644 --- a/view/theme/frio/templates/admin/site.tpl +++ b/view/theme/frio/templates/admin/site.tpl @@ -275,9 +275,7 @@ {{include file="field_input.tpl" field=$maxloadavg}} {{include file="field_input.tpl" field=$min_memory}} {{include file="field_input.tpl" field=$worker_queues}} - {{include file="field_checkbox.tpl" field=$worker_dont_fork}} {{include file="field_checkbox.tpl" field=$worker_fastlane}} - {{include file="field_checkbox.tpl" field=$worker_frontend}}
From 3aeaa4ea82704d77ded7036499ce45c9409c0b99 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Thu, 7 Jan 2021 09:20:03 -0500 Subject: [PATCH 2/2] Add test for program execution function installer check --- tests/src/Core/InstallerTest.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/src/Core/InstallerTest.php b/tests/src/Core/InstallerTest.php index 0933d9e1a..936c6ddac 100644 --- a/tests/src/Core/InstallerTest.php +++ b/tests/src/Core/InstallerTest.php @@ -93,6 +93,8 @@ class InstallerTest extends MockedTest $this->mockL10nT('Error: JSON PHP module required but not installed.', 1); $this->mockL10nT('File Information PHP module', 1); $this->mockL10nT('Error: File Information PHP module required but not installed.', 1); + $this->mockL10nT('Program execution functions', 1); + $this->mockL10nT('Error: Program execution functions required but not enabled.', 1); } private function assertCheckExist($position, $title, $help, $status, $required, $assertionArray) @@ -232,10 +234,20 @@ class InstallerTest extends MockedTest $install->getChecks()); $this->mockFunctionL10TCalls(); - $this->setFunctions(['json_encode' => false]); + $this->setFunctions(['proc_open' => false]); $install = new Installer(); self::assertFalse($install->checkFunctions()); self::assertCheckExist(9, + 'Program execution functions', + 'Error: Program execution functions required but not enabled.', + false, + true, + $install->getChecks()); + $this->mockFunctionL10TCalls(); + $this->setFunctions(['json_encode' => false]); + $install = new Installer(); + self::assertFalse($install->checkFunctions()); + self::assertCheckExist(10, 'JSON PHP module', 'Error: JSON PHP module required but not installed.', false, @@ -246,7 +258,7 @@ class InstallerTest extends MockedTest $this->setFunctions(['finfo_open' => false]); $install = new Installer(); self::assertFalse($install->checkFunctions()); - self::assertCheckExist(10, + self::assertCheckExist(11, 'File Information PHP module', 'Error: File Information PHP module required but not installed.', false,