Remove the queue from the core

pull/6978/head
Michael 4 years ago
parent bd13a73b2b
commit 46a99a05a1
  1. 2
      config/dbstructure.config.php
  2. 5
      doc/Addons.md
  3. 5
      doc/de/Addons.md
  4. 54
      mod/admin.php
  5. 2
      src/Core/Console/ArchiveContact.php
  6. 126
      src/Model/Queue.php
  7. 32
      src/Protocol/Diaspora.php
  8. 3
      src/Worker/Cron.php
  9. 164
      src/Worker/Queue.php
  10. 30
      update.php
  11. 2
      view/templates/admin/summary.tpl
  12. 2
      view/theme/frio/templates/admin/summary.tpl

@ -34,7 +34,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1308);
define('DB_UPDATE_VERSION', 1309);
}
return [

@ -685,11 +685,6 @@ Here is a complete list of all hook callbacks with file locations (as of 24-Sep-
Hook::callAll('notifier_end', $target_item);
### src/Worker/Queue.php
Hook::callAll('queue_predeliver', $r);
Hook::callAll('queue_deliver', $params);
### src/Module/Login.php
Hook::callAll('authenticate', $addon_auth);

@ -434,11 +434,6 @@ Eine komplette Liste aller Hook-Callbacks mit den zugehörigen Dateien (am 01-Ap
Hook::callAll('notifier_end', $target_item);
### src/Worker/Queue.php
Hook::callAll('queue_predeliver', $r);
Hook::callAll('queue_deliver', $params);
### src/Module/Login.php
Hook::callAll('authenticate', $addon_auth);

@ -205,7 +205,6 @@ function admin_content(App $a)
'tos' => ['admin/tos/' , L10n::t('Terms of Service') , 'tos']]],
'database' => [L10n::t('Database'), [
'dbsync' => ['admin/dbsync/' , L10n::t('DB updates') , 'dbsync'],
'queue' => ['admin/queue/' , L10n::t('Inspect Queue') , 'queue'],
'deferred' => ['admin/deferred/' , L10n::t('Inspect Deferred Workers'), 'deferred'],
'workerqueue' => ['admin/workerqueue/' , L10n::t('Inspect worker Queue') , 'workerqueue']]],
'tools' => [L10n::t('Tools'), [
@ -274,9 +273,6 @@ function admin_content(App $a)
case 'dbsync':
$o = admin_page_dbsync($a);
break;
case 'queue':
$o = admin_page_queue($a);
break;
case 'deferred':
$o = admin_page_workerqueue($a, true);
break;
@ -789,52 +785,6 @@ function admin_page_federation(App $a)
]);
}
/**
* @brief Admin Inspect Queue Page
*
* Generates a page for the admin to have a look into the current queue of
* postings that are not deliverable. Shown are the name and url of the
* recipient, the delivery network and the dates when the posting was generated
* and the last time tried to deliver the posting.
*
* The returned string holds the content of the page.
*
* @param App $a
* @return string
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
function admin_page_queue(App $a)
{
// get content from the queue table
$entries = DBA::p("SELECT `contact`.`name`, `contact`.`nurl`,
`queue`.`id`, `queue`.`network`, `queue`.`created`, `queue`.`last`
FROM `queue` INNER JOIN `contact` ON `contact`.`id` = `queue`.`cid`
ORDER BY `queue`.`cid`, `queue`.`created`");
$r = [];
while ($entry = DBA::fetch($entries)) {
$entry['created'] = DateTimeFormat::local($entry['created']);
$entry['last'] = DateTimeFormat::local($entry['last']);
$r[] = $entry;
}
DBA::close($entries);
$t = Renderer::getMarkupTemplate('admin/queue.tpl');
return Renderer::replaceMacros($t, [
'$title' => L10n::t('Administration'),
'$page' => L10n::t('Inspect Queue'),
'$count' => count($r),
'id_header' => L10n::t('ID'),
'$to_header' => L10n::t('Recipient Name'),
'$url_header' => L10n::t('Recipient Profile'),
'$network_header' => L10n::t('Network'),
'$created_header' => L10n::t('Created'),
'$last_header' => L10n::t('Last Tried'),
'$info' => L10n::t('This page lists the content of the queue for outgoing postings. These are postings the initial delivery failed for. They will be resend later and eventually deleted if the delivery fails permanently.'),
'$entries' => $r,
]);
}
/**
* @brief Admin Inspect Worker Queue Page
*
@ -977,8 +927,6 @@ function admin_page_summary(App $a)
$pending = Register::getPendingCount();
$queue = DBA::count('queue', []);
$deferred = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?",
DBA::NULL_DATETIME, DateTimeFormat::utcNow()]);
@ -987,7 +935,7 @@ function admin_page_summary(App $a)
// We can do better, but this is a quick queue status
$queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'deferred' => $deferred, 'workerq' => $workerqueue];
$queues = ['label' => L10n::t('Message queues'), 'deferred' => $deferred, 'workerq' => $workerqueue];
$r = q("SHOW variables LIKE 'max_allowed_packet'");

@ -65,8 +65,6 @@ HELP;
throw new RuntimeException(L10n::t('Could not find any unarchived contact entry for this URL (%s)', $nurl));
}
if (DBA::update('contact', ['archive' => true], ['nurl' => $nurl])) {
$condition = ["`cid` IN (SELECT `id` FROM `contact` WHERE `archive`)"];
DBA::delete('queue', $condition);
$this->out(L10n::t('The contact entries have been archived'));
} else {
throw new RuntimeException('The contact archival failed.');

@ -1,126 +0,0 @@
<?php
/**
* @file src/Model/Queue.php
*/
namespace Friendica\Model;
use Friendica\Core\Config;
use Friendica\Core\Logger;
use Friendica\Database\DBA;
use Friendica\Util\DateTimeFormat;
class Queue
{
/**
* @param string $id id
* @throws \Exception
*/
public static function updateTime($id)
{
Logger::log('queue: requeue item ' . $id);
$queue = DBA::selectFirst('queue', ['retrial'], ['id' => $id]);
if (!DBA::isResult($queue)) {
return;
}
$retrial = $queue['retrial'];
if ($retrial > 14) {
self::removeItem($id);
}
// Calculate the delay until the next trial
$delay = (($retrial + 3) ** 4) + (rand(1, 30) * ($retrial + 1));
$next = DateTimeFormat::utc('now + ' . $delay . ' seconds');
DBA::update('queue', ['last' => DateTimeFormat::utcNow(), 'retrial' => $retrial + 1, 'next' => $next], ['id' => $id]);
}
/**
* @param string $id id
* @throws \Exception
*/
public static function removeItem($id)
{
Logger::log('queue: remove queue item ' . $id);
DBA::delete('queue', ['id' => $id]);
}
/**
* @brief Checks if the communication with a given contact had problems recently
*
* @param int $cid Contact id
*
* @return bool The communication with this contact has currently problems
* @throws \Exception
*/
public static function wasDelayed($cid)
{
// Are there queue entries that were recently added?
$r = q("SELECT `id` FROM `queue` WHERE `cid` = %d
AND `last` > UTC_TIMESTAMP() - INTERVAL 15 MINUTE LIMIT 1",
intval($cid)
);
$was_delayed = DBA::isResult($r);
// We set "term-date" to a current date if the communication has problems.
// If the communication works again we reset this value.
if ($was_delayed) {
$r = q("SELECT `term-date` FROM `contact` WHERE `id` = %d AND `term-date` <= '1000-01-01' LIMIT 1",
intval($cid)
);
$was_delayed = !DBA::isResult($r);
}
return $was_delayed;
}
/**
* @param string $cid cid
* @param string $network network
* @param string $msg message
* @param boolean $batch batch, default false
* @param string $guid
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
public static function add($cid, $network, $msg, $batch = false, $guid = '')
{
$max_queue = Config::get('system', 'max_contact_queue');
if ($max_queue < 1) {
$max_queue = 500;
}
$batch_queue = Config::get('system', 'max_batch_queue');
if ($batch_queue < 1) {
$batch_queue = 1000;
}
$r = q("SELECT COUNT(*) AS `total` FROM `queue` INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
WHERE `queue`.`cid` = %d AND `contact`.`self` = 0 ",
intval($cid)
);
if (DBA::isResult($r)) {
if ($batch && ($r[0]['total'] > $batch_queue)) {
Logger::log('too many queued items for batch server ' . $cid . ' - discarding message');
return;
} elseif ((! $batch) && ($r[0]['total'] > $max_queue)) {
Logger::log('too many queued items for contact ' . $cid . ' - discarding message');
return;
}
}
DBA::insert('queue', [
'cid' => $cid,
'network' => $network,
'guid' => $guid,
'created' => DateTimeFormat::utcNow(),
'last' => DateTimeFormat::utcNow(),
'content' => $msg,
'batch' =>($batch) ? 1 : 0
]);
Logger::log('Added item ' . $guid . ' for ' . $cid);
}
}

@ -28,7 +28,6 @@ use Friendica\Model\GContact;
use Friendica\Model\Group;
use Friendica\Model\Item;
use Friendica\Model\Profile;
use Friendica\Model\Queue;
use Friendica\Model\User;
use Friendica\Network\Probe;
use Friendica\Util\Crypto;
@ -3170,15 +3169,14 @@ class Diaspora
* @param array $contact Target of the communication
* @param string $envelope The message that is to be transmitted
* @param bool $public_batch Is it a public post?
* @param bool $queue_run Is the transmission called from the queue?
* @param string $guid message guid
* @param bool $no_queue
* @param bool $no_defer Don't defer a failing delivery
*
* @return int Result of the transmission
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
*/
public static function transmit(array $owner, array $contact, $envelope, $public_batch, $queue_run = false, $guid = "", $no_queue = false)
public static function transmit(array $owner, array $contact, $envelope, $public_batch, $guid = "", $no_defer = false)
{
$enabled = intval(Config::get("system", "diaspora_enabled"));
if (!$enabled) {
@ -3205,24 +3203,20 @@ class Diaspora
Logger::log("transmit: ".$logid."-".$guid." ".$dest_url);
if (!$queue_run && Queue::wasDelayed($contact["id"])) {
$return_code = 0;
} else {
if (!intval(Config::get("system", "diaspora_test"))) {
$content_type = (($public_batch) ? "application/magic-envelope+xml" : "application/json");
if (!intval(Config::get("system", "diaspora_test"))) {
$content_type = (($public_batch) ? "application/magic-envelope+xml" : "application/json");
$postResult = Network::post($dest_url."/", $envelope, ["Content-Type: ".$content_type]);
$return_code = $postResult->getReturnCode();
} else {
Logger::log("test_mode");
return 200;
}
$postResult = Network::post($dest_url."/", $envelope, ["Content-Type: ".$content_type]);
$return_code = $postResult->getReturnCode();
} else {
Logger::log("test_mode");
return 200;
}
Logger::log("transmit: ".$logid."-".$guid." to ".$dest_url." returns: ".$return_code);
if (!$return_code || (($return_code == 503) && (stristr($postResult->getHeader(), "retry-after")))) {
if (!$no_queue && !empty($contact['contact-type']) && ($contact['contact-type'] != Contact::TYPE_RELAY)) {
if (!$no_defer && !empty($contact['contact-type']) && ($contact['contact-type'] != Contact::TYPE_RELAY)) {
Logger::info('defer message', ['log' => $logid, 'guid' => $guid, 'destination' => $dest_url]);
// defer message for redelivery
Worker::defer();
@ -3263,13 +3257,13 @@ class Diaspora
* @param array $message The message data
* @param bool $public_batch Is it a public post?
* @param string $guid message guid
* @param bool $no_queue
* @param bool $no_defer Don't defer a failing delivery
*
* @return int Result of the transmission
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
*/
private static function buildAndTransmit(array $owner, array $contact, $type, $message, $public_batch = false, $guid = "", $no_queue = false)
private static function buildAndTransmit(array $owner, array $contact, $type, $message, $public_batch = false, $guid = "", $no_defer = false)
{
$msg = self::buildPostXml($type, $message);
@ -3283,7 +3277,7 @@ class Diaspora
$envelope = self::buildMessage($msg, $owner, $contact, $owner['uprvkey'], $contact['pubkey'], $public_batch);
$return_code = self::transmit($owner, $contact, $envelope, $public_batch, false, $guid, $no_queue);
$return_code = self::transmit($owner, $contact, $envelope, $public_batch, $guid, $no_defer);
Logger::log("guid: ".$guid." result ".$return_code, Logger::DEBUG);

@ -47,9 +47,6 @@ class Cron
// Fork the cron jobs in separate parts to avoid problems when one of them is crashing
Hook::fork($a->queue['priority'], "cron");
// run queue delivery process in the background
Worker::add(PRIORITY_NEGLIGIBLE, "Queue");
// run the process to discover global contacts in the background
Worker::add(PRIORITY_LOW, "DiscoverPoCo");

@ -1,164 +0,0 @@
<?php
/**
* @file src/Worker/Queue.php
*/
namespace Friendica\Worker;
use Friendica\Core\Cache;
use Friendica\Core\Config;
use Friendica\Core\Hook;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\Model\Contact;
use Friendica\Model\PushSubscriber;
use Friendica\Model\Queue as QueueModel;
use Friendica\Model\User;
use Friendica\Protocol\DFRN;
use Friendica\Protocol\Diaspora;
use Friendica\Protocol\PortableContact;
use Friendica\Protocol\Salmon;
class Queue
{
public static function execute($queue_id = 0)
{
$cachekey_deadguy = 'queue_run:deadguy:';
$cachekey_server = 'queue_run:server:';
$no_dead_check = Config::get('system', 'queue_no_dead_check', false);
if (!$queue_id) {
Logger::log('filling queue jobs - start');
// Handling the pubsubhubbub requests
PushSubscriber::requeue();
$r = DBA::toArray(DBA::p("SELECT `id` FROM `queue` WHERE `next` < UTC_TIMESTAMP() ORDER BY `batch`, `cid`"));
Hook::callAll('queue_predeliver', $r);
if (DBA::isResult($r)) {
foreach ($r as $q_item) {
Logger::log('Call queue for id ' . $q_item['id']);
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], "Queue", (int) $q_item['id']);
}
}
Logger::log('filling queue jobs - end');
return;
}
// delivering
$q_item = DBA::selectFirst('queue', [], ['id' => $queue_id]);
if (!DBA::isResult($q_item)) {
return;
}
$contact = DBA::selectFirst('contact', [], ['id' => $q_item['cid']]);
if (!DBA::isResult($contact)) {
QueueModel::removeItem($q_item['id']);
return;
}
if (empty($contact['notify']) || $contact['archive']) {
QueueModel::removeItem($q_item['id']);
return;
}
$dead = Cache::get($cachekey_deadguy . $contact['notify']);
if (!is_null($dead) && $dead && !$no_dead_check) {
Logger::log('queue: skipping known dead url: ' . $contact['notify']);
QueueModel::updateTime($q_item['id']);
return;
}
if (!$no_dead_check) {
$server = PortableContact::detectServer($contact['url']);
if ($server != "") {
$vital = Cache::get($cachekey_server . $server);
if (is_null($vital)) {
Logger::log("Check server " . $server . " (" . $contact["network"] . ")");
$vital = PortableContact::checkServer($server, $contact["network"], true);
Cache::set($cachekey_server . $server, $vital, Cache::MINUTE);
}
if (!is_null($vital) && !$vital) {
Logger::log('queue: skipping dead server: ' . $server);
QueueModel::updateTime($q_item['id']);
return;
}
}
}
$user = DBA::selectFirst('user', [], ['uid' => $contact['uid']]);
if (!DBA::isResult($user)) {
QueueModel::removeItem($q_item['id']);
return;
}
$data = $q_item['content'];
$public = $q_item['batch'];
$owner = User::getOwnerDataById($user['uid']);
$deliver_status = 0;
switch ($contact['network']) {
case Protocol::DFRN:
Logger::log('queue: dfrndelivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>');
$deliver_status = DFRN::deliver($owner, $contact, $data);
if (($deliver_status >= 200) && ($deliver_status <= 299)) {
QueueModel::removeItem($q_item['id']);
} else {
QueueModel::updateTime($q_item['id']);
Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE);
}
break;
case Protocol::OSTATUS:
Logger::log('queue: slapdelivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>');
$deliver_status = Salmon::slapper($owner, $contact['notify'], $data);
if ($deliver_status == -1) {
QueueModel::updateTime($q_item['id']);
Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE);
} else {
QueueModel::removeItem($q_item['id']);
}
break;
case Protocol::DIASPORA:
Logger::log('queue: diaspora_delivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>');
$deliver_status = Diaspora::transmit($owner, $contact, $data, $public, true, 'Queue:' . $q_item['id'], true);
if ((($deliver_status >= 200) && ($deliver_status <= 299)) ||
($contact['contact-type'] == Contact::TYPE_RELAY)) {
QueueModel::removeItem($q_item['id']);
} else {
QueueModel::updateTime($q_item['id']);
Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE);
}
break;
default:
$params = ['owner' => $owner, 'contact' => $contact, 'queue' => $q_item, 'result' => false];
Hook::callAll('queue_deliver', $params);
if ($params['result']) {
QueueModel::removeItem($q_item['id']);
} else {
QueueModel::updateTime($q_item['id']);
}
break;
}
Logger::log('Deliver status ' . (int)$deliver_status . ' for item ' . $q_item['id'] . ' to ' . $contact['name'] . ' <' . $contact['url'] . '>');
return;
}
}

@ -13,6 +13,7 @@ use Friendica\Model\GContact;
use Friendica\Model\Item;
use Friendica\Model\User;
use Friendica\Util\DateTimeFormat;
use Friendica\Worker\Delivery;
/**
*
@ -334,7 +335,7 @@ function update_1298()
DBA::update('profile', [$translateKey => $key], ['id' => $data['id']]);
Logger::notice('Updated contact', ['action' => 'update', 'contact' => $data['id'], "$translateKey" => $key,
'was' => $data[$translateKey]]);
Worker::add(PRIORITY_LOW, 'ProfileUpdate', $data['id']);
Worker::add(PRIORITY_LOW, 'ProfileUpdate', $data['id']);
Contact::updateSelfFromUserID($data['id']);
GContact::updateForUser($data['id']);
$success++;
@ -346,3 +347,30 @@ function update_1298()
}
return Update::SUCCESS;
}
function update_1309()
{
$queue = DBA::select('queue', ['id', 'cid', 'guid']);
while ($entry = DBA::fetch($queue)) {
$contact = DBA::selectFirst('contact', ['uid'], ['id' => $entry['cid']]);
if (!DBA::isResult($contact)) {
continue;
}
$item = Item::selectFirst(['id', 'gravity'], ['uid' => $contact['uid'], 'guid' => $entry['guid']]);
if (!DBA::isResult($item)) {
continue;
}
if ($item['gravity'] == GRAVITY_PARENT) {
$cmd = Delivery::POST;
} else {
$cmd = Delivery::COMMENT;
}
$deliver_options = ['priority' => PRIORITY_MEDIUM, 'dont_fork' => true];
Worker::add($deliver_options, 'Delivery', $cmd, $item['id'], $entry['cid']);
Logger::info('Added delivery worker', ['command' => $cmd, 'item' => $item['id'], 'contact' => $entry['cid']]);
}
return Update::SUCCESS;
}

@ -11,7 +11,7 @@
<dl>
<dt>{{$queues.label}}</dt>
<dd><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></dd>
<dd><a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></dd>
</dl>
<dl>
<dt>{{$pending.0}}</dt>

@ -14,7 +14,7 @@
{{* The work queues short statistic. *}}
<div id="admin-summary-queues" class="col-lg-12 col-md-12 col-sm-12 col-xs-12 admin-summary">
<div class="col-lg-4 col-md-4 col-sm-4 col-xs-12 admin-summary-label-name text-muted">{{$queues.label}}</div>
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 admin-summary-entry"><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></div>
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 admin-summary-entry"><a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></div>
</div>
{{* Number of pending registrations. *}}

Loading…
Cancel
Save