From 95dc03092638b70d76e574f9cb7b4c72e41d1f92 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 23 Oct 2018 20:38:28 +0000 Subject: [PATCH 1/4] Show deferred worker tasks in the admin interface and workerqueue logs --- mod/admin.php | 86 ++++++++++++--------- src/Core/Worker.php | 17 +++- view/templates/admin/summary.tpl | 2 +- view/theme/frio/templates/admin/summary.tpl | 2 +- 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/mod/admin.php b/mod/admin.php index e3b537fcf..c50927e97 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -181,32 +181,33 @@ function admin_content(App $a) // array(url, name, extra css classes) // not part of $aside to make the template more adjustable $aside_sub = [ - 'information' => [ L10n::t('Information'), [ - "overview" => ["admin/", L10n::t("Overview"), "overview" ], - 'federation' => ["admin/federation/" , L10n::t('Federation Statistics'), "federation"] ]], - 'configuration' => [ L10n::t('Configuration'), [ - 'site' => ["admin/site/" , L10n::t("Site") , "site"], - 'users' => ["admin/users/" , L10n::t("Users") , "users"], - 'addons' => ["admin/addons/" , L10n::t("Addons") , "addons"], - 'themes' => ["admin/themes/" , L10n::t("Themes") , "themes"], - 'features' => ["admin/features/" , L10n::t("Additional features") , "features"], - 'tos' => ["admin/tos/" , L10n::t("Terms of Service") , "tos"] ]], - 'database' => [ L10n::t('Database'), [ - 'dbsync' => ["admin/dbsync/" , L10n::t('DB updates') , "dbsync"], - 'queue' => ["admin/queue/" , L10n::t('Inspect Queue') , "queue"], - 'workerqueue' => ["admin/workerqueue/" , L10n::t('Inspect worker Queue') , "workerqueue"] ]], - 'tools' => [ L10n::t('Tools'), [ - 'contactblock' => ["admin/contactblock/", L10n::t('Contact Blocklist') , "contactblock"], - 'blocklist' => ["admin/blocklist/" , L10n::t('Server Blocklist') , "blocklist"], - 'deleteitem' => ["admin/deleteitem/" , L10n::t('Delete Item') , 'deleteitem'], ]], - "logs" => [ L10n::t("Logs"), [ - "logsconfig" => ["admin/logs/", L10n::t("Logs"), "logs"], - "logsview" => ["admin/viewlogs/", L10n::t("View Logs"), 'viewlogs'] + 'information' => [L10n::t('Information'), [ + 'overview' => ['admin/', L10n::t('Overview'), 'overview'], + 'federation' => ['admin/federation/' , L10n::t('Federation Statistics'), 'federation']]], + 'configuration' => [L10n::t('Configuration'), [ + 'site' => ['admin/site/' , L10n::t('Site') , 'site'], + 'users' => ['admin/users/' , L10n::t('Users') , 'users'], + 'addons' => ['admin/addons/' , L10n::t('Addons') , 'addons'], + 'themes' => ['admin/themes/' , L10n::t('Themes') , 'themes'], + 'features' => ['admin/features/' , L10n::t('Additional features') , 'features'], + 'tos' => ['admin/tos/' , L10n::t('Terms of Service') , 'tos']]], + 'database' => [L10n::t('Database'), [ + 'dbsync' => ['admin/dbsync/' , L10n::t('DB updates') , 'dbsync'], + 'queue' => ['admin/queue/' , L10n::t('Inspect Queue') , 'queue'], + 'deferred' => ['admin/deferred/' , L10n::t('Inspect Deferred Workers'), 'deferred'], + 'workerqueue' => ['admin/workerqueue/' , L10n::t('Inspect worker Queue') , 'workerqueue']]], + 'tools' => [L10n::t('Tools'), [ + 'contactblock' => ['admin/contactblock/', L10n::t('Contact Blocklist') , 'contactblock'], + 'blocklist' => ['admin/blocklist/' , L10n::t('Server Blocklist') , 'blocklist'], + 'deleteitem' => ['admin/deleteitem/' , L10n::t('Delete Item') , 'deleteitem'],]], + 'logs' => [L10n::t('Logs'), [ + 'logsconfig' => ['admin/logs/', L10n::t('Logs'), 'logs'], + 'logsview' => ['admin/viewlogs/', L10n::t('View Logs'), 'viewlogs'] ]], - "diagnostics" => [ L10n::t("Diagnostics"), [ - "phpinfo" => ['phpinfo/', L10n::t('PHP Info'), 'phpinfo'], - "probe" => ['probe/', L10n::t('probe address'), 'probe'], - "webfinger" =>['webfinger/', L10n::t('check webfinger'), 'webfinger'] + 'diagnostics' => [L10n::t('Diagnostics'), [ + 'phpinfo' => ['phpinfo/', L10n::t('PHP Info'), 'phpinfo'], + 'probe' => ['probe/', L10n::t('probe address'), 'probe'], + 'webfinger' =>['webfinger/', L10n::t('check webfinger'), 'webfinger'] ]] ]; @@ -264,8 +265,11 @@ function admin_content(App $a) case 'queue': $o = admin_page_queue($a); break; + case 'deferred': + $o = admin_page_workerqueue($a, true); + break; case 'workerqueue': - $o = admin_page_workerqueue($a); + $o = admin_page_workerqueue($a, false); break; case 'federation': $o = admin_page_federation($a); @@ -790,10 +794,20 @@ function admin_page_queue(App $a) * @param App $a * @return string */ -function admin_page_workerqueue(App $a) +function admin_page_workerqueue(App $a, $deferred) { // get jobs from the workerqueue table - $entries = DBA::select('workerqueue', ['id', 'parameter', 'created', 'priority'], ['done' => 0], ['order'=> ['priority']]); + if ($deferred) { + $condition = ["NOT `done` AND `next_try` > ?", DateTimeFormat::utcNow()]; + $sub_title = L10n::t('Inspect Deferred Worker Queue'); + $info = L10n::t("This page lists the deferred worker jobs. This are jobs that couldn't be executed at the first time."); + } else { + $condition = ["NOT `done` AND `next_try` < ?", DateTimeFormat::utcNow()]; + $sub_title = L10n::t('Inspect Worker Queue'); + $info = L10n::t('This page lists the currently queued worker jobs. These jobs are handled by the worker cronjob you\'ve set up during install.'); + } + + $entries = DBA::select('workerqueue', ['id', 'parameter', 'created', 'priority'], $condition, ['order'=> ['priority']]); $r = []; while ($entry = DBA::fetch($entries)) { @@ -807,13 +821,13 @@ function admin_page_workerqueue(App $a) $t = get_markup_template('admin/workerqueue.tpl'); return replace_macros($t, [ '$title' => L10n::t('Administration'), - '$page' => L10n::t('Inspect Worker Queue'), + '$page' => $sub_title, '$count' => count($r), '$id_header' => L10n::t('ID'), '$param_header' => L10n::t('Job Parameters'), '$created_header' => L10n::t('Created'), '$prio_header' => L10n::t('Priority'), - '$info' => L10n::t('This page lists the currently queued worker jobs. These jobs are handled by the worker cronjob you\'ve set up during install.'), + '$info' => $info, '$entries' => $r, ]); } @@ -900,15 +914,17 @@ function admin_page_summary(App $a) $pending = Register::getPendingCount(); - $r = q("SELECT COUNT(*) AS `total` FROM `queue` WHERE 1"); - $queue = (($r) ? $r[0]['total'] : 0); + $queue = DBA::count('queue', []); - $r = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE NOT `done`"); - $workerqueue = (($r) ? $r[0]['total'] : 0); + $deferred = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?", + DBA::NULL_DATETIME, DateTimeFormat::utcNow()]); + + $workerqueue = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` < ?", + DBA::NULL_DATETIME, DateTimeFormat::utcNow()]); // We can do better, but this is a quick queue status - $queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'workerq' => $workerqueue]; + $queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'deferred' => $deferred, 'workerq' => $workerqueue]; $r = q("SHOW variables LIKE 'max_allowed_packet'"); diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 76400d469..a3c588460 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -146,6 +146,17 @@ class Worker logger("Couldn't select a workerqueue entry, quitting process " . getmypid() . ".", LOGGER_DEBUG); } + /** + * @brief Returns the number of deferred entries in the worker queue + * + * @return integer Number of deferred entries in the worker queue + */ + private static function deferredEntries() + { + return DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?", + DBA::NULL_DATETIME, DateTimeFormat::utcNow()]); + } + /** * @brief Returns the number of non executed entries in the worker queue * @@ -679,6 +690,7 @@ class Worker } $entries = self::totalEntries(); + $deferred = self::deferredEntries(); if (Config::get("system", "worker_fastlane", false) && ($queues > 0) && ($entries > 0) && ($active >= $queues)) { $top_priority = self::highestPriority(); @@ -690,7 +702,7 @@ class Worker } } - logger("Load: ".$load."/".$maxsysload." - processes: ".$active."/".$entries.$processlist." - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG); + logger("Load: " . $load ."/" . $maxsysload . " - processes: " . $deferred . "/" . $active . "/" . $entries . $processlist . " - maximum: " . $queues . "/" . $maxqueues, LOGGER_DEBUG); // Are there fewer workers running as possible? Then fork a new one. if (!Config::get("system", "worker_dont_fork", false) && ($queues > ($active + 1)) && ($entries > 1)) { @@ -797,13 +809,14 @@ class Worker $queue_length = Config::get('system', 'worker_fetch_limit', 1); $lower_job_limit = $worker_queues * $queue_length * 2; $jobs = self::totalEntries(); + $deferred = self::deferredEntries(); // Now do some magic $exponent = 2; $slope = $queue_length / pow($lower_job_limit, $exponent); $limit = min($queue_length, ceil($slope * pow($jobs, $exponent))); - logger('Total: '.$jobs.' - Maximum: '.$queue_length.' - jobs per queue: '.$limit, LOGGER_DEBUG); + logger('Deferred: ' . $deferred . ' - Total: ' . $jobs . ' - Maximum: ' . $queue_length . ' - jobs per queue: ' . $limit, LOGGER_DEBUG); $ids = []; if (self::passingSlow($highest_priority)) { // Are there waiting processes with a higher priority than the currently highest? diff --git a/view/templates/admin/summary.tpl b/view/templates/admin/summary.tpl index 202fd43b3..298097f79 100644 --- a/view/templates/admin/summary.tpl +++ b/view/templates/admin/summary.tpl @@ -11,7 +11,7 @@
{{$queues.label}}
-
{{$queues.queue}} - {{$queues.workerq}}
+
{{$queues.queue}} - {{$queues.deferred}} - {{$queues.workerq}}
{{$pending.0}}
diff --git a/view/theme/frio/templates/admin/summary.tpl b/view/theme/frio/templates/admin/summary.tpl index 3c33f92fd..66afda721 100644 --- a/view/theme/frio/templates/admin/summary.tpl +++ b/view/theme/frio/templates/admin/summary.tpl @@ -14,7 +14,7 @@ {{* The work queues short statistic. *}} {{* Number of pending registrations. *}} From c0a24d80dda2dccf44579b1ae0ddba87abead658 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 24 Oct 2018 04:46:45 +0000 Subject: [PATCH 2/4] Remove items more memory friendly --- src/Worker/RemoveContact.php | 10 ++++++++++ src/Worker/RemoveUser.php | 9 ++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Worker/RemoveContact.php b/src/Worker/RemoveContact.php index 8f986eab1..8485adda6 100644 --- a/src/Worker/RemoveContact.php +++ b/src/Worker/RemoveContact.php @@ -7,6 +7,7 @@ namespace Friendica\Worker; use Friendica\Database\DBA; use Friendica\Core\Protocol; +use Friendica\Model\Item; require_once 'include/dba.php'; @@ -21,6 +22,15 @@ class RemoveContact { } // Now we delete the contact and all depending tables + $condition = ['contact-id' => $id]; + do { + $items = Item::select(['id'], $condition, ['limit' => 100]); + while ($item = Item::fetch($items)) { + DBA::delete('item', ['id' => $item['id']]); + } + DBA::close($items); + } while (Item::exists($condition)); + DBA::delete('contact', ['id' => $id]); } } diff --git a/src/Worker/RemoveUser.php b/src/Worker/RemoveUser.php index dfa5ccc09..d8966e5a4 100644 --- a/src/Worker/RemoveUser.php +++ b/src/Worker/RemoveUser.php @@ -20,6 +20,13 @@ class RemoveUser { } // Now we delete all user items - Item::delete(['uid' => $uid], PRIORITY_LOW); + $condition = ['uid' => $uid, 'deleted' => false]; + do { + $items = Item::select(['id'], $condition, ['limit' => 100]); + while ($item = Item::fetch($items)) { + Item::deleteById($item['id'], PRIORITY_LOW); + } + DBA::close($items); + } while (Item::exists($condition)); } } From a6f6db8d4ec9406b93689aaf4a1da2e2524e42c9 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 24 Oct 2018 04:50:27 +0000 Subject: [PATCH 3/4] Improve query speed --- src/Worker/RemoveContact.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Worker/RemoveContact.php b/src/Worker/RemoveContact.php index 8485adda6..2e3f16db3 100644 --- a/src/Worker/RemoveContact.php +++ b/src/Worker/RemoveContact.php @@ -16,13 +16,13 @@ class RemoveContact { // Only delete if the contact is to be deleted $condition = ['network' => Protocol::PHANTOM, 'id' => $id]; - $r = DBA::exists('contact', $condition); - if (!DBA::isResult($r)) { + $contact = DBA::selectFirst('contact', ['uid'], $condition); + if (!DBA::isResult($contact)) { return; } // Now we delete the contact and all depending tables - $condition = ['contact-id' => $id]; + $condition = ['uid' => $contact['uid'], 'contact-id' => $id]; do { $items = Item::select(['id'], $condition, ['limit' => 100]); while ($item = Item::fetch($items)) { From c25d60e107cd4d067ba3107cb8d623a76d5bdf81 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 24 Oct 2018 04:51:37 +0000 Subject: [PATCH 4/4] Removed finished work from the comments --- src/Protocol/ActivityPub/Processor.php | 5 +---- src/Protocol/ActivityPub/Transmitter.php | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 9bafb5e17..8a80bd2be 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -17,10 +17,7 @@ use Friendica\Core\Config; use Friendica\Protocol\ActivityPub; /** - * ActivityPub Protocol class - * - * To-Do: - * - Store Diaspora signature + * ActivityPub Processor Protocol class */ class Processor { diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index a37d03537..b0e6cd362 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -39,9 +39,6 @@ require_once 'include/api.php'; * * Complicated object types: * - Undo Announce - * - * General: - * - Queueing unsucessful deliveries */ class Transmitter {