Merge pull request #6978 from annando/no-queue

Remove the queue from the core
This commit is contained in:
Hypolite Petovan 2019-04-07 08:05:51 -04:00 committed by GitHub
commit 9ed3f3d6d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 48 additions and 421 deletions

View File

@ -34,7 +34,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1308);
define('DB_UPDATE_VERSION', 1309);
}
return [
@ -1118,26 +1118,6 @@ return [
"next_try" => ["next_try"],
]
],
"queue" => [
"comment" => "Queue for messages that couldn't be delivered",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"],
"cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "Message receiver"],
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Receiver's network"],
"guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Unique GUID of the message"],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date, when the message was created"],
"last" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of last trial"],
"next" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Next retrial date"],
"retrial" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Retrial counter"],
"content" => ["type" => "mediumtext", "comment" => ""],
"batch" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
],
"indexes" => [
"PRIMARY" => ["id"],
"last" => ["last"],
"next" => ["next"],
]
],
"register" => [
"comment" => "registrations requiring admin approval",
"fields" => [

View File

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2019.06-dev (Dalmatian Bellflower)
-- DB_UPDATE_VERSION 1308
-- DB_UPDATE_VERSION 1309
-- ------------------------------------------
@ -1027,25 +1027,6 @@ CREATE TABLE IF NOT EXISTS `push_subscriber` (
INDEX `next_try` (`next_try`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Used for OStatus: Contains feed subscribers';
--
-- TABLE queue
--
CREATE TABLE IF NOT EXISTS `queue` (
`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
`cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Message receiver',
`network` char(4) NOT NULL DEFAULT '' COMMENT 'Receiver\'s network',
`guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'Unique GUID of the message',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date, when the message was created',
`last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last trial',
`next` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Next retrial date',
`retrial` tinyint NOT NULL DEFAULT 0 COMMENT 'Retrial counter',
`content` mediumtext COMMENT '',
`batch` boolean NOT NULL DEFAULT '0' COMMENT '',
PRIMARY KEY(`id`),
INDEX `last` (`last`),
INDEX `next` (`next`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Queue for messages that couldn\'t be delivered';
--
-- TABLE register
--

View File

@ -690,11 +690,6 @@ Here is a complete list of all hook callbacks with file locations (as of 24-Sep-
Hook::callAll('notifier_end', $target_item);
### src/Worker/Queue.php
Hook::callAll('queue_predeliver', $r);
Hook::callAll('queue_deliver', $params);
### src/Module/Login.php
Hook::callAll('authenticate', $addon_auth);

View File

@ -434,11 +434,6 @@ Eine komplette Liste aller Hook-Callbacks mit den zugehörigen Dateien (am 01-Ap
Hook::callAll('notifier_end', $target_item);
### src/Worker/Queue.php
Hook::callAll('queue_predeliver', $r);
Hook::callAll('queue_deliver', $params);
### src/Module/Login.php
Hook::callAll('authenticate', $addon_auth);

View File

@ -205,7 +205,6 @@ function admin_content(App $a)
'tos' => ['admin/tos/' , L10n::t('Terms of Service') , 'tos']]],
'database' => [L10n::t('Database'), [
'dbsync' => ['admin/dbsync/' , L10n::t('DB updates') , 'dbsync'],
'queue' => ['admin/queue/' , L10n::t('Inspect Queue') , 'queue'],
'deferred' => ['admin/deferred/' , L10n::t('Inspect Deferred Workers'), 'deferred'],
'workerqueue' => ['admin/workerqueue/' , L10n::t('Inspect worker Queue') , 'workerqueue']]],
'tools' => [L10n::t('Tools'), [
@ -274,9 +273,6 @@ function admin_content(App $a)
case 'dbsync':
$o = admin_page_dbsync($a);
break;
case 'queue':
$o = admin_page_queue($a);
break;
case 'deferred':
$o = admin_page_workerqueue($a, true);
break;
@ -789,52 +785,6 @@ function admin_page_federation(App $a)
]);
}
/**
* @brief Admin Inspect Queue Page
*
* Generates a page for the admin to have a look into the current queue of
* postings that are not deliverable. Shown are the name and url of the
* recipient, the delivery network and the dates when the posting was generated
* and the last time tried to deliver the posting.
*
* The returned string holds the content of the page.
*
* @param App $a
* @return string
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
function admin_page_queue(App $a)
{
// get content from the queue table
$entries = DBA::p("SELECT `contact`.`name`, `contact`.`nurl`,
`queue`.`id`, `queue`.`network`, `queue`.`created`, `queue`.`last`
FROM `queue` INNER JOIN `contact` ON `contact`.`id` = `queue`.`cid`
ORDER BY `queue`.`cid`, `queue`.`created`");
$r = [];
while ($entry = DBA::fetch($entries)) {
$entry['created'] = DateTimeFormat::local($entry['created']);
$entry['last'] = DateTimeFormat::local($entry['last']);
$r[] = $entry;
}
DBA::close($entries);
$t = Renderer::getMarkupTemplate('admin/queue.tpl');
return Renderer::replaceMacros($t, [
'$title' => L10n::t('Administration'),
'$page' => L10n::t('Inspect Queue'),
'$count' => count($r),
'id_header' => L10n::t('ID'),
'$to_header' => L10n::t('Recipient Name'),
'$url_header' => L10n::t('Recipient Profile'),
'$network_header' => L10n::t('Network'),
'$created_header' => L10n::t('Created'),
'$last_header' => L10n::t('Last Tried'),
'$info' => L10n::t('This page lists the content of the queue for outgoing postings. These are postings the initial delivery failed for. They will be resend later and eventually deleted if the delivery fails permanently.'),
'$entries' => $r,
]);
}
/**
* @brief Admin Inspect Worker Queue Page
*
@ -977,8 +927,6 @@ function admin_page_summary(App $a)
$pending = Register::getPendingCount();
$queue = DBA::count('queue', []);
$deferred = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?",
DBA::NULL_DATETIME, DateTimeFormat::utcNow()]);
@ -987,7 +935,7 @@ function admin_page_summary(App $a)
// We can do better, but this is a quick queue status
$queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'deferred' => $deferred, 'workerq' => $workerqueue];
$queues = ['label' => L10n::t('Message queues'), 'deferred' => $deferred, 'workerq' => $workerqueue];
$r = q("SHOW variables LIKE 'max_allowed_packet'");

View File

@ -65,8 +65,6 @@ HELP;
throw new RuntimeException(L10n::t('Could not find any unarchived contact entry for this URL (%s)', $nurl));
}
if (DBA::update('contact', ['archive' => true], ['nurl' => $nurl])) {
$condition = ["`cid` IN (SELECT `id` FROM `contact` WHERE `archive`)"];
DBA::delete('queue', $condition);
$this->out(L10n::t('The contact entries have been archived'));
} else {
throw new RuntimeException('The contact archival failed.');

View File

@ -1,126 +0,0 @@
<?php
/**
* @file src/Model/Queue.php
*/
namespace Friendica\Model;
use Friendica\Core\Config;
use Friendica\Core\Logger;
use Friendica\Database\DBA;
use Friendica\Util\DateTimeFormat;
class Queue
{
/**
* @param string $id id
* @throws \Exception
*/
public static function updateTime($id)
{
Logger::log('queue: requeue item ' . $id);
$queue = DBA::selectFirst('queue', ['retrial'], ['id' => $id]);
if (!DBA::isResult($queue)) {
return;
}
$retrial = $queue['retrial'];
if ($retrial > 14) {
self::removeItem($id);
}
// Calculate the delay until the next trial
$delay = (($retrial + 3) ** 4) + (rand(1, 30) * ($retrial + 1));
$next = DateTimeFormat::utc('now + ' . $delay . ' seconds');
DBA::update('queue', ['last' => DateTimeFormat::utcNow(), 'retrial' => $retrial + 1, 'next' => $next], ['id' => $id]);
}
/**
* @param string $id id
* @throws \Exception
*/
public static function removeItem($id)
{
Logger::log('queue: remove queue item ' . $id);
DBA::delete('queue', ['id' => $id]);
}
/**
* @brief Checks if the communication with a given contact had problems recently
*
* @param int $cid Contact id
*
* @return bool The communication with this contact has currently problems
* @throws \Exception
*/
public static function wasDelayed($cid)
{
// Are there queue entries that were recently added?
$r = q("SELECT `id` FROM `queue` WHERE `cid` = %d
AND `last` > UTC_TIMESTAMP() - INTERVAL 15 MINUTE LIMIT 1",
intval($cid)
);
$was_delayed = DBA::isResult($r);
// We set "term-date" to a current date if the communication has problems.
// If the communication works again we reset this value.
if ($was_delayed) {
$r = q("SELECT `term-date` FROM `contact` WHERE `id` = %d AND `term-date` <= '1000-01-01' LIMIT 1",
intval($cid)
);
$was_delayed = !DBA::isResult($r);
}
return $was_delayed;
}
/**
* @param string $cid cid
* @param string $network network
* @param string $msg message
* @param boolean $batch batch, default false
* @param string $guid
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
public static function add($cid, $network, $msg, $batch = false, $guid = '')
{
$max_queue = Config::get('system', 'max_contact_queue');
if ($max_queue < 1) {
$max_queue = 500;
}
$batch_queue = Config::get('system', 'max_batch_queue');
if ($batch_queue < 1) {
$batch_queue = 1000;
}
$r = q("SELECT COUNT(*) AS `total` FROM `queue` INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
WHERE `queue`.`cid` = %d AND `contact`.`self` = 0 ",
intval($cid)
);
if (DBA::isResult($r)) {
if ($batch && ($r[0]['total'] > $batch_queue)) {
Logger::log('too many queued items for batch server ' . $cid . ' - discarding message');
return;
} elseif ((! $batch) && ($r[0]['total'] > $max_queue)) {
Logger::log('too many queued items for contact ' . $cid . ' - discarding message');
return;
}
}
DBA::insert('queue', [
'cid' => $cid,
'network' => $network,
'guid' => $guid,
'created' => DateTimeFormat::utcNow(),
'last' => DateTimeFormat::utcNow(),
'content' => $msg,
'batch' =>($batch) ? 1 : 0
]);
Logger::log('Added item ' . $guid . ' for ' . $cid);
}
}

View File

@ -28,7 +28,6 @@ use Friendica\Model\GContact;
use Friendica\Model\Group;
use Friendica\Model\Item;
use Friendica\Model\Profile;
use Friendica\Model\Queue;
use Friendica\Model\User;
use Friendica\Network\Probe;
use Friendica\Util\Crypto;
@ -3170,15 +3169,14 @@ class Diaspora
* @param array $contact Target of the communication
* @param string $envelope The message that is to be transmitted
* @param bool $public_batch Is it a public post?
* @param bool $queue_run Is the transmission called from the queue?
* @param string $guid message guid
* @param bool $no_queue
* @param bool $no_defer Don't defer a failing delivery
*
* @return int Result of the transmission
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
*/
public static function transmit(array $owner, array $contact, $envelope, $public_batch, $queue_run = false, $guid = "", $no_queue = false)
public static function transmit(array $owner, array $contact, $envelope, $public_batch, $guid = "", $no_defer = false)
{
$enabled = intval(Config::get("system", "diaspora_enabled"));
if (!$enabled) {
@ -3205,9 +3203,6 @@ class Diaspora
Logger::log("transmit: ".$logid."-".$guid." ".$dest_url);
if (!$queue_run && Queue::wasDelayed($contact["id"])) {
$return_code = 0;
} else {
if (!intval(Config::get("system", "diaspora_test"))) {
$content_type = (($public_batch) ? "application/magic-envelope+xml" : "application/json");
@ -3217,12 +3212,11 @@ class Diaspora
Logger::log("test_mode");
return 200;
}
}
Logger::log("transmit: ".$logid."-".$guid." to ".$dest_url." returns: ".$return_code);
if (!$return_code || (($return_code == 503) && (stristr($postResult->getHeader(), "retry-after")))) {
if (!$no_queue && !empty($contact['contact-type']) && ($contact['contact-type'] != Contact::TYPE_RELAY)) {
if (!$no_defer && !empty($contact['contact-type']) && ($contact['contact-type'] != Contact::TYPE_RELAY)) {
Logger::info('defer message', ['log' => $logid, 'guid' => $guid, 'destination' => $dest_url]);
// defer message for redelivery
Worker::defer();
@ -3263,13 +3257,13 @@ class Diaspora
* @param array $message The message data
* @param bool $public_batch Is it a public post?
* @param string $guid message guid
* @param bool $no_queue
* @param bool $no_defer Don't defer a failing delivery
*
* @return int Result of the transmission
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
*/
private static function buildAndTransmit(array $owner, array $contact, $type, $message, $public_batch = false, $guid = "", $no_queue = false)
private static function buildAndTransmit(array $owner, array $contact, $type, $message, $public_batch = false, $guid = "", $no_defer = false)
{
$msg = self::buildPostXml($type, $message);
@ -3283,7 +3277,7 @@ class Diaspora
$envelope = self::buildMessage($msg, $owner, $contact, $owner['uprvkey'], $contact['pubkey'], $public_batch);
$return_code = self::transmit($owner, $contact, $envelope, $public_batch, false, $guid, $no_queue);
$return_code = self::transmit($owner, $contact, $envelope, $public_batch, $guid, $no_defer);
Logger::log("guid: ".$guid." result ".$return_code, Logger::DEBUG);

View File

@ -47,9 +47,6 @@ class Cron
// Fork the cron jobs in separate parts to avoid problems when one of them is crashing
Hook::fork($a->queue['priority'], "cron");
// run queue delivery process in the background
Worker::add(PRIORITY_NEGLIGIBLE, "Queue");
// run the process to discover global contacts in the background
Worker::add(PRIORITY_LOW, "DiscoverPoCo");

View File

@ -1,164 +0,0 @@
<?php
/**
* @file src/Worker/Queue.php
*/
namespace Friendica\Worker;
use Friendica\Core\Cache;
use Friendica\Core\Config;
use Friendica\Core\Hook;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\Model\Contact;
use Friendica\Model\PushSubscriber;
use Friendica\Model\Queue as QueueModel;
use Friendica\Model\User;
use Friendica\Protocol\DFRN;
use Friendica\Protocol\Diaspora;
use Friendica\Protocol\PortableContact;
use Friendica\Protocol\Salmon;
class Queue
{
public static function execute($queue_id = 0)
{
$cachekey_deadguy = 'queue_run:deadguy:';
$cachekey_server = 'queue_run:server:';
$no_dead_check = Config::get('system', 'queue_no_dead_check', false);
if (!$queue_id) {
Logger::log('filling queue jobs - start');
// Handling the pubsubhubbub requests
PushSubscriber::requeue();
$r = DBA::toArray(DBA::p("SELECT `id` FROM `queue` WHERE `next` < UTC_TIMESTAMP() ORDER BY `batch`, `cid`"));
Hook::callAll('queue_predeliver', $r);
if (DBA::isResult($r)) {
foreach ($r as $q_item) {
Logger::log('Call queue for id ' . $q_item['id']);
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], "Queue", (int) $q_item['id']);
}
}
Logger::log('filling queue jobs - end');
return;
}
// delivering
$q_item = DBA::selectFirst('queue', [], ['id' => $queue_id]);
if (!DBA::isResult($q_item)) {
return;
}
$contact = DBA::selectFirst('contact', [], ['id' => $q_item['cid']]);
if (!DBA::isResult($contact)) {
QueueModel::removeItem($q_item['id']);
return;
}
if (empty($contact['notify']) || $contact['archive']) {
QueueModel::removeItem($q_item['id']);
return;
}
$dead = Cache::get($cachekey_deadguy . $contact['notify']);
if (!is_null($dead) && $dead && !$no_dead_check) {
Logger::log('queue: skipping known dead url: ' . $contact['notify']);
QueueModel::updateTime($q_item['id']);
return;
}
if (!$no_dead_check) {
$server = PortableContact::detectServer($contact['url']);
if ($server != "") {
$vital = Cache::get($cachekey_server . $server);
if (is_null($vital)) {
Logger::log("Check server " . $server . " (" . $contact["network"] . ")");
$vital = PortableContact::checkServer($server, $contact["network"], true);
Cache::set($cachekey_server . $server, $vital, Cache::MINUTE);
}
if (!is_null($vital) && !$vital) {
Logger::log('queue: skipping dead server: ' . $server);
QueueModel::updateTime($q_item['id']);
return;
}
}
}
$user = DBA::selectFirst('user', [], ['uid' => $contact['uid']]);
if (!DBA::isResult($user)) {
QueueModel::removeItem($q_item['id']);
return;
}
$data = $q_item['content'];
$public = $q_item['batch'];
$owner = User::getOwnerDataById($user['uid']);
$deliver_status = 0;
switch ($contact['network']) {
case Protocol::DFRN:
Logger::log('queue: dfrndelivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>');
$deliver_status = DFRN::deliver($owner, $contact, $data);
if (($deliver_status >= 200) && ($deliver_status <= 299)) {
QueueModel::removeItem($q_item['id']);
} else {
QueueModel::updateTime($q_item['id']);
Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE);
}
break;
case Protocol::OSTATUS:
Logger::log('queue: slapdelivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>');
$deliver_status = Salmon::slapper($owner, $contact['notify'], $data);
if ($deliver_status == -1) {
QueueModel::updateTime($q_item['id']);
Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE);
} else {
QueueModel::removeItem($q_item['id']);
}
break;
case Protocol::DIASPORA:
Logger::log('queue: diaspora_delivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>');
$deliver_status = Diaspora::transmit($owner, $contact, $data, $public, true, 'Queue:' . $q_item['id'], true);
if ((($deliver_status >= 200) && ($deliver_status <= 299)) ||
($contact['contact-type'] == Contact::TYPE_RELAY)) {
QueueModel::removeItem($q_item['id']);
} else {
QueueModel::updateTime($q_item['id']);
Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE);
}
break;
default:
$params = ['owner' => $owner, 'contact' => $contact, 'queue' => $q_item, 'result' => false];
Hook::callAll('queue_deliver', $params);
if ($params['result']) {
QueueModel::removeItem($q_item['id']);
} else {
QueueModel::updateTime($q_item['id']);
}
break;
}
Logger::log('Deliver status ' . (int)$deliver_status . ' for item ' . $q_item['id'] . ' to ' . $contact['name'] . ' <' . $contact['url'] . '>');
return;
}
}

View File

@ -13,6 +13,7 @@ use Friendica\Model\GContact;
use Friendica\Model\Item;
use Friendica\Model\User;
use Friendica\Util\DateTimeFormat;
use Friendica\Worker\Delivery;
/**
*
@ -346,3 +347,31 @@ function update_1298()
}
return Update::SUCCESS;
}
function update_1309()
{
$queue = DBA::select('queue', ['id', 'cid', 'guid']);
while ($entry = DBA::fetch($queue)) {
$contact = DBA::selectFirst('contact', ['uid'], ['id' => $entry['cid']]);
if (!DBA::isResult($contact)) {
continue;
}
$item = Item::selectFirst(['id', 'gravity'], ['uid' => $contact['uid'], 'guid' => $entry['guid']]);
if (!DBA::isResult($item)) {
continue;
}
if ($item['gravity'] == GRAVITY_PARENT) {
$cmd = Delivery::POST;
} else {
$cmd = Delivery::COMMENT;
}
$deliver_options = ['priority' => PRIORITY_MEDIUM, 'dont_fork' => true];
Worker::add($deliver_options, 'Delivery', $cmd, $item['id'], $entry['cid']);
Logger::info('Added delivery worker', ['command' => $cmd, 'item' => $item['id'], 'contact' => $entry['cid']]);
DBA::delete('queue', ['id' => $entry['id']]);
}
return Update::SUCCESS;
}

View File

@ -11,7 +11,7 @@
<dl>
<dt>{{$queues.label}}</dt>
<dd><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></dd>
<dd><a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></dd>
</dl>
<dl>
<dt>{{$pending.0}}</dt>

View File

@ -14,7 +14,7 @@
{{* The work queues short statistic. *}}
<div id="admin-summary-queues" class="col-lg-12 col-md-12 col-sm-12 col-xs-12 admin-summary">
<div class="col-lg-4 col-md-4 col-sm-4 col-xs-12 admin-summary-label-name text-muted">{{$queues.label}}</div>
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 admin-summary-entry"><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></div>
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 admin-summary-entry"><a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></div>
</div>
{{* Number of pending registrations. *}}