tobias/friendica
1
0
Fork 0
Code Pull requests Activity

Merge pull request #6014 from annando/deferred-tasks

Browse source 
Show deferred worker tasks in the admin interface and workerqueue logs
This commit is contained in:
Hypolite Petovan 2018-10-23 17:39:31 -04:00 committed by GitHub
parent e6fd05d8e8 95dc030926
commit 7538dd2084
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 68 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
mod/admin.php
View file

@ -181,32 +181,33 @@ function admin_content(App $a)
// array(url, name, extra css classes)
// not part of $aside to make the template more adjustable
$aside_sub = [
'information' => [ L10n::t('Information'), [
"overview" => ["admin/", L10n::t("Overview"), "overview" ],
'federation' => ["admin/federation/" , L10n::t('Federation Statistics'), "federation"] ]],
'configuration' => [ L10n::t('Configuration'), [
'site' => ["admin/site/" , L10n::t("Site") , "site"],
'users' => ["admin/users/" , L10n::t("Users") , "users"],
'addons' => ["admin/addons/" , L10n::t("Addons") , "addons"],
'themes' => ["admin/themes/" , L10n::t("Themes") , "themes"],
'features' => ["admin/features/" , L10n::t("Additional features") , "features"],
'tos' => ["admin/tos/" , L10n::t("Terms of Service") , "tos"] ]],
'database' => [ L10n::t('Database'), [
'dbsync' => ["admin/dbsync/" , L10n::t('DB updates') , "dbsync"],
'queue' => ["admin/queue/" , L10n::t('Inspect Queue') , "queue"],
'workerqueue' => ["admin/workerqueue/" , L10n::t('Inspect worker Queue') , "workerqueue"] ]],
'tools' => [ L10n::t('Tools'), [
'contactblock' => ["admin/contactblock/", L10n::t('Contact Blocklist') , "contactblock"],
'blocklist' => ["admin/blocklist/" , L10n::t('Server Blocklist') , "blocklist"],
'deleteitem' => ["admin/deleteitem/" , L10n::t('Delete Item') , 'deleteitem'], ]],
"logs" => [ L10n::t("Logs"), [
"logsconfig" => ["admin/logs/", L10n::t("Logs"), "logs"],
"logsview" => ["admin/viewlogs/", L10n::t("View Logs"), 'viewlogs']
'information' => [L10n::t('Information'), [
'overview' => ['admin/', L10n::t('Overview'), 'overview'],
'federation' => ['admin/federation/' , L10n::t('Federation Statistics'), 'federation']]],
'configuration' => [L10n::t('Configuration'), [
'site' => ['admin/site/' , L10n::t('Site') , 'site'],
'users' => ['admin/users/' , L10n::t('Users') , 'users'],
'addons' => ['admin/addons/' , L10n::t('Addons') , 'addons'],
'themes' => ['admin/themes/' , L10n::t('Themes') , 'themes'],
'features' => ['admin/features/' , L10n::t('Additional features') , 'features'],
'tos' => ['admin/tos/' , L10n::t('Terms of Service') , 'tos']]],
'database' => [L10n::t('Database'), [
'dbsync' => ['admin/dbsync/' , L10n::t('DB updates') , 'dbsync'],
'queue' => ['admin/queue/' , L10n::t('Inspect Queue') , 'queue'],
'deferred' => ['admin/deferred/' , L10n::t('Inspect Deferred Workers'), 'deferred'],
'workerqueue' => ['admin/workerqueue/' , L10n::t('Inspect worker Queue') , 'workerqueue']]],
'tools' => [L10n::t('Tools'), [
'contactblock' => ['admin/contactblock/', L10n::t('Contact Blocklist') , 'contactblock'],
'blocklist' => ['admin/blocklist/' , L10n::t('Server Blocklist') , 'blocklist'],
'deleteitem' => ['admin/deleteitem/' , L10n::t('Delete Item') , 'deleteitem'],]],
'logs' => [L10n::t('Logs'), [
'logsconfig' => ['admin/logs/', L10n::t('Logs'), 'logs'],
'logsview' => ['admin/viewlogs/', L10n::t('View Logs'), 'viewlogs']
]],
"diagnostics" => [ L10n::t("Diagnostics"), [
"phpinfo" => ['phpinfo/', L10n::t('PHP Info'), 'phpinfo'],
"probe" => ['probe/', L10n::t('probe address'), 'probe'],
"webfinger" =>['webfinger/', L10n::t('check webfinger'), 'webfinger']
'diagnostics' => [L10n::t('Diagnostics'), [
'phpinfo' => ['phpinfo/', L10n::t('PHP Info'), 'phpinfo'],
'probe' => ['probe/', L10n::t('probe address'), 'probe'],
'webfinger' =>['webfinger/', L10n::t('check webfinger'), 'webfinger']
]]
];
@ -264,8 +265,11 @@ function admin_content(App $a)
case 'queue':
$o = admin_page_queue($a);
break;
case 'deferred':
$o = admin_page_workerqueue($a, true);
break;
case 'workerqueue':
$o = admin_page_workerqueue($a);
$o = admin_page_workerqueue($a, false);
break;
case 'federation':
$o = admin_page_federation($a);
@ -790,10 +794,20 @@ function admin_page_queue(App $a)
* @param App $a
* @return string
*/
function admin_page_workerqueue(App $a)
function admin_page_workerqueue(App $a, $deferred)
{
// get jobs from the workerqueue table
$entries = DBA::select('workerqueue', ['id', 'parameter', 'created', 'priority'], ['done' => 0], ['order'=> ['priority']]);
if ($deferred) {
$condition = ["NOT `done` AND `next_try` > ?", DateTimeFormat::utcNow()];
$sub_title = L10n::t('Inspect Deferred Worker Queue');
$info = L10n::t("This page lists the deferred worker jobs. This are jobs that couldn't be executed at the first time.");
} else {
$condition = ["NOT `done` AND `next_try` < ?", DateTimeFormat::utcNow()];
$sub_title = L10n::t('Inspect Worker Queue');
$info = L10n::t('This page lists the currently queued worker jobs. These jobs are handled by the worker cronjob you\'ve set up during install.');
}
$entries = DBA::select('workerqueue', ['id', 'parameter', 'created', 'priority'], $condition, ['order'=> ['priority']]);
$r = [];
while ($entry = DBA::fetch($entries)) {
@ -807,13 +821,13 @@ function admin_page_workerqueue(App $a)
$t = get_markup_template('admin/workerqueue.tpl');
return replace_macros($t, [
'$title' => L10n::t('Administration'),
'$page' => L10n::t('Inspect Worker Queue'),
'$page' => $sub_title,
'$count' => count($r),
'$id_header' => L10n::t('ID'),
'$param_header' => L10n::t('Job Parameters'),
'$created_header' => L10n::t('Created'),
'$prio_header' => L10n::t('Priority'),
'$info' => L10n::t('This page lists the currently queued worker jobs. These jobs are handled by the worker cronjob you\'ve set up during install.'),
'$info' => $info,
'$entries' => $r,
]);
}
@ -900,15 +914,17 @@ function admin_page_summary(App $a)
$pending = Register::getPendingCount();
$r = q("SELECT COUNT(*) AS `total` FROM `queue` WHERE 1");
$queue = (($r) ? $r[0]['total'] : 0);
$queue = DBA::count('queue', []);
$r = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE NOT `done`");
$workerqueue = (($r) ? $r[0]['total'] : 0);
$deferred = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?",
DBA::NULL_DATETIME, DateTimeFormat::utcNow()]);
$workerqueue = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` < ?",
DBA::NULL_DATETIME, DateTimeFormat::utcNow()]);
// We can do better, but this is a quick queue status
$queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'workerq' => $workerqueue];
$queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'deferred' => $deferred, 'workerq' => $workerqueue];
$r = q("SHOW variables LIKE 'max_allowed_packet'");

17
src/Core/Worker.php
View file

@ -146,6 +146,17 @@ class Worker
logger("Couldn't select a workerqueue entry, quitting process " . getmypid() . ".", LOGGER_DEBUG);
}
/**
* @brief Returns the number of deferred entries in the worker queue
*
* @return integer Number of deferred entries in the worker queue
*/
private static function deferredEntries()
{
return DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?",
DBA::NULL_DATETIME, DateTimeFormat::utcNow()]);
}
/**
* @brief Returns the number of non executed entries in the worker queue
*
@ -679,6 +690,7 @@ class Worker
}
$entries = self::totalEntries();
$deferred = self::deferredEntries();
if (Config::get("system", "worker_fastlane", false) && ($queues > 0) && ($entries > 0) && ($active >= $queues)) {
$top_priority = self::highestPriority();
@ -690,7 +702,7 @@ class Worker
}
}
logger("Load: ".$load."/".$maxsysload." - processes: ".$active."/".$entries.$processlist." - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG);
logger("Load: " . $load ."/" . $maxsysload . " - processes: " . $deferred . "/" . $active . "/" . $entries . $processlist . " - maximum: " . $queues . "/" . $maxqueues, LOGGER_DEBUG);
// Are there fewer workers running as possible? Then fork a new one.
if (!Config::get("system", "worker_dont_fork", false) && ($queues > ($active + 1)) && ($entries > 1)) {
@ -797,13 +809,14 @@ class Worker
$queue_length = Config::get('system', 'worker_fetch_limit', 1);
$lower_job_limit = $worker_queues * $queue_length * 2;
$jobs = self::totalEntries();
$deferred = self::deferredEntries();
// Now do some magic
$exponent = 2;
$slope = $queue_length / pow($lower_job_limit, $exponent);
$limit = min($queue_length, ceil($slope * pow($jobs, $exponent)));
logger('Total: '.$jobs.' - Maximum: '.$queue_length.' - jobs per queue: '.$limit, LOGGER_DEBUG);
logger('Deferred: ' . $deferred . ' - Total: ' . $jobs . ' - Maximum: ' . $queue_length . ' - jobs per queue: ' . $limit, LOGGER_DEBUG);
$ids = [];
if (self::passingSlow($highest_priority)) {
// Are there waiting processes with a higher priority than the currently highest?

2
view/templates/admin/summary.tpl
View file

@ -11,7 +11,7 @@
<dl>
<dt>{{$queues.label}}</dt>
<dd><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></dd>
<dd><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}} - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></dd>
</dl>
<dl>
<dt>{{$pending.0}}</dt>

2
view/theme/frio/templates/admin/summary.tpl
View file

@ -14,7 +14,7 @@
{{* The work queues short statistic. *}}
<div id="admin-summary-queues" class="col-lg-12 col-md-12 col-sm-12 col-xs-12 admin-summary">
<div class="col-lg-4 col-md-4 col-sm-4 col-xs-12 admin-summary-label-name text-muted">{{$queues.label}}</div>
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 admin-summary-entry"><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></div>
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 admin-summary-entry"><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}} - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></div>
</div>
{{* Number of pending registrations. *}}