diff --git a/config/dbstructure.config.php b/config/dbstructure.config.php index 4c2afc7f0..8ae14ad41 100755 --- a/config/dbstructure.config.php +++ b/config/dbstructure.config.php @@ -34,7 +34,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1308); + define('DB_UPDATE_VERSION', 1309); } return [ @@ -1118,26 +1118,6 @@ return [ "next_try" => ["next_try"], ] ], - "queue" => [ - "comment" => "Queue for messages that couldn't be delivered", - "fields" => [ - "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], - "cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "Message receiver"], - "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Receiver's network"], - "guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Unique GUID of the message"], - "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date, when the message was created"], - "last" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of last trial"], - "next" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Next retrial date"], - "retrial" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Retrial counter"], - "content" => ["type" => "mediumtext", "comment" => ""], - "batch" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], - ], - "indexes" => [ - "PRIMARY" => ["id"], - "last" => ["last"], - "next" => ["next"], - ] - ], "register" => [ "comment" => "registrations requiring admin approval", "fields" => [ diff --git a/database.sql b/database.sql index 5ba700bd8..9c4af9a43 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2019.06-dev (Dalmatian Bellflower) --- DB_UPDATE_VERSION 1308 +-- DB_UPDATE_VERSION 1309 -- ------------------------------------------ @@ -1027,25 +1027,6 @@ CREATE TABLE IF NOT EXISTS `push_subscriber` ( INDEX `next_try` (`next_try`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Used for OStatus: Contains feed subscribers'; --- --- TABLE queue --- -CREATE TABLE IF NOT EXISTS `queue` ( - `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', - `cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Message receiver', - `network` char(4) NOT NULL DEFAULT '' COMMENT 'Receiver\'s network', - `guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'Unique GUID of the message', - `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date, when the message was created', - `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last trial', - `next` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Next retrial date', - `retrial` tinyint NOT NULL DEFAULT 0 COMMENT 'Retrial counter', - `content` mediumtext COMMENT '', - `batch` boolean NOT NULL DEFAULT '0' COMMENT '', - PRIMARY KEY(`id`), - INDEX `last` (`last`), - INDEX `next` (`next`) -) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Queue for messages that couldn\'t be delivered'; - -- -- TABLE register -- diff --git a/doc/Addons.md b/doc/Addons.md index b47c32e6d..17d58fb2c 100644 --- a/doc/Addons.md +++ b/doc/Addons.md @@ -690,11 +690,6 @@ Here is a complete list of all hook callbacks with file locations (as of 24-Sep- Hook::callAll('notifier_end', $target_item); -### src/Worker/Queue.php - - Hook::callAll('queue_predeliver', $r); - Hook::callAll('queue_deliver', $params); - ### src/Module/Login.php Hook::callAll('authenticate', $addon_auth); diff --git a/doc/de/Addons.md b/doc/de/Addons.md index d85d4d07b..35ce0e28b 100644 --- a/doc/de/Addons.md +++ b/doc/de/Addons.md @@ -434,11 +434,6 @@ Eine komplette Liste aller Hook-Callbacks mit den zugehörigen Dateien (am 01-Ap Hook::callAll('notifier_end', $target_item); -### src/Worker/Queue.php - - Hook::callAll('queue_predeliver', $r); - Hook::callAll('queue_deliver', $params); - ### src/Module/Login.php Hook::callAll('authenticate', $addon_auth); diff --git a/mod/admin.php b/mod/admin.php index 5fc53427c..0df907cba 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -205,7 +205,6 @@ function admin_content(App $a) 'tos' => ['admin/tos/' , L10n::t('Terms of Service') , 'tos']]], 'database' => [L10n::t('Database'), [ 'dbsync' => ['admin/dbsync/' , L10n::t('DB updates') , 'dbsync'], - 'queue' => ['admin/queue/' , L10n::t('Inspect Queue') , 'queue'], 'deferred' => ['admin/deferred/' , L10n::t('Inspect Deferred Workers'), 'deferred'], 'workerqueue' => ['admin/workerqueue/' , L10n::t('Inspect worker Queue') , 'workerqueue']]], 'tools' => [L10n::t('Tools'), [ @@ -274,9 +273,6 @@ function admin_content(App $a) case 'dbsync': $o = admin_page_dbsync($a); break; - case 'queue': - $o = admin_page_queue($a); - break; case 'deferred': $o = admin_page_workerqueue($a, true); break; @@ -789,52 +785,6 @@ function admin_page_federation(App $a) ]); } -/** - * @brief Admin Inspect Queue Page - * - * Generates a page for the admin to have a look into the current queue of - * postings that are not deliverable. Shown are the name and url of the - * recipient, the delivery network and the dates when the posting was generated - * and the last time tried to deliver the posting. - * - * The returned string holds the content of the page. - * - * @param App $a - * @return string - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ -function admin_page_queue(App $a) -{ - // get content from the queue table - $entries = DBA::p("SELECT `contact`.`name`, `contact`.`nurl`, - `queue`.`id`, `queue`.`network`, `queue`.`created`, `queue`.`last` - FROM `queue` INNER JOIN `contact` ON `contact`.`id` = `queue`.`cid` - ORDER BY `queue`.`cid`, `queue`.`created`"); - - $r = []; - while ($entry = DBA::fetch($entries)) { - $entry['created'] = DateTimeFormat::local($entry['created']); - $entry['last'] = DateTimeFormat::local($entry['last']); - $r[] = $entry; - } - DBA::close($entries); - - $t = Renderer::getMarkupTemplate('admin/queue.tpl'); - return Renderer::replaceMacros($t, [ - '$title' => L10n::t('Administration'), - '$page' => L10n::t('Inspect Queue'), - '$count' => count($r), - 'id_header' => L10n::t('ID'), - '$to_header' => L10n::t('Recipient Name'), - '$url_header' => L10n::t('Recipient Profile'), - '$network_header' => L10n::t('Network'), - '$created_header' => L10n::t('Created'), - '$last_header' => L10n::t('Last Tried'), - '$info' => L10n::t('This page lists the content of the queue for outgoing postings. These are postings the initial delivery failed for. They will be resend later and eventually deleted if the delivery fails permanently.'), - '$entries' => $r, - ]); -} - /** * @brief Admin Inspect Worker Queue Page * @@ -977,8 +927,6 @@ function admin_page_summary(App $a) $pending = Register::getPendingCount(); - $queue = DBA::count('queue', []); - $deferred = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?", DBA::NULL_DATETIME, DateTimeFormat::utcNow()]); @@ -987,7 +935,7 @@ function admin_page_summary(App $a) // We can do better, but this is a quick queue status - $queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'deferred' => $deferred, 'workerq' => $workerqueue]; + $queues = ['label' => L10n::t('Message queues'), 'deferred' => $deferred, 'workerq' => $workerqueue]; $r = q("SHOW variables LIKE 'max_allowed_packet'"); diff --git a/src/Core/Console/ArchiveContact.php b/src/Core/Console/ArchiveContact.php index 27de85901..b93c079b9 100644 --- a/src/Core/Console/ArchiveContact.php +++ b/src/Core/Console/ArchiveContact.php @@ -65,8 +65,6 @@ HELP; throw new RuntimeException(L10n::t('Could not find any unarchived contact entry for this URL (%s)', $nurl)); } if (DBA::update('contact', ['archive' => true], ['nurl' => $nurl])) { - $condition = ["`cid` IN (SELECT `id` FROM `contact` WHERE `archive`)"]; - DBA::delete('queue', $condition); $this->out(L10n::t('The contact entries have been archived')); } else { throw new RuntimeException('The contact archival failed.'); diff --git a/src/Model/Queue.php b/src/Model/Queue.php deleted file mode 100644 index 355433492..000000000 --- a/src/Model/Queue.php +++ /dev/null @@ -1,126 +0,0 @@ - $id]); - if (!DBA::isResult($queue)) { - return; - } - - $retrial = $queue['retrial']; - - if ($retrial > 14) { - self::removeItem($id); - } - - // Calculate the delay until the next trial - $delay = (($retrial + 3) ** 4) + (rand(1, 30) * ($retrial + 1)); - $next = DateTimeFormat::utc('now + ' . $delay . ' seconds'); - - DBA::update('queue', ['last' => DateTimeFormat::utcNow(), 'retrial' => $retrial + 1, 'next' => $next], ['id' => $id]); - } - - /** - * @param string $id id - * @throws \Exception - */ - public static function removeItem($id) - { - Logger::log('queue: remove queue item ' . $id); - DBA::delete('queue', ['id' => $id]); - } - - /** - * @brief Checks if the communication with a given contact had problems recently - * - * @param int $cid Contact id - * - * @return bool The communication with this contact has currently problems - * @throws \Exception - */ - public static function wasDelayed($cid) - { - // Are there queue entries that were recently added? - $r = q("SELECT `id` FROM `queue` WHERE `cid` = %d - AND `last` > UTC_TIMESTAMP() - INTERVAL 15 MINUTE LIMIT 1", - intval($cid) - ); - - $was_delayed = DBA::isResult($r); - - // We set "term-date" to a current date if the communication has problems. - // If the communication works again we reset this value. - if ($was_delayed) { - $r = q("SELECT `term-date` FROM `contact` WHERE `id` = %d AND `term-date` <= '1000-01-01' LIMIT 1", - intval($cid) - ); - $was_delayed = !DBA::isResult($r); - } - - return $was_delayed; - } - - /** - * @param string $cid cid - * @param string $network network - * @param string $msg message - * @param boolean $batch batch, default false - * @param string $guid - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function add($cid, $network, $msg, $batch = false, $guid = '') - { - - $max_queue = Config::get('system', 'max_contact_queue'); - if ($max_queue < 1) { - $max_queue = 500; - } - - $batch_queue = Config::get('system', 'max_batch_queue'); - if ($batch_queue < 1) { - $batch_queue = 1000; - } - - $r = q("SELECT COUNT(*) AS `total` FROM `queue` INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id` - WHERE `queue`.`cid` = %d AND `contact`.`self` = 0 ", - intval($cid) - ); - - if (DBA::isResult($r)) { - if ($batch && ($r[0]['total'] > $batch_queue)) { - Logger::log('too many queued items for batch server ' . $cid . ' - discarding message'); - return; - } elseif ((! $batch) && ($r[0]['total'] > $max_queue)) { - Logger::log('too many queued items for contact ' . $cid . ' - discarding message'); - return; - } - } - - DBA::insert('queue', [ - 'cid' => $cid, - 'network' => $network, - 'guid' => $guid, - 'created' => DateTimeFormat::utcNow(), - 'last' => DateTimeFormat::utcNow(), - 'content' => $msg, - 'batch' =>($batch) ? 1 : 0 - ]); - Logger::log('Added item ' . $guid . ' for ' . $cid); - } -} diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 7e8511711..33cbb2659 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -28,7 +28,6 @@ use Friendica\Model\GContact; use Friendica\Model\Group; use Friendica\Model\Item; use Friendica\Model\Profile; -use Friendica\Model\Queue; use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Util\Crypto; @@ -3170,15 +3169,14 @@ class Diaspora * @param array $contact Target of the communication * @param string $envelope The message that is to be transmitted * @param bool $public_batch Is it a public post? - * @param bool $queue_run Is the transmission called from the queue? * @param string $guid message guid - * @param bool $no_queue + * @param bool $no_defer Don't defer a failing delivery * * @return int Result of the transmission * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function transmit(array $owner, array $contact, $envelope, $public_batch, $queue_run = false, $guid = "", $no_queue = false) + public static function transmit(array $owner, array $contact, $envelope, $public_batch, $guid = "", $no_defer = false) { $enabled = intval(Config::get("system", "diaspora_enabled")); if (!$enabled) { @@ -3205,24 +3203,20 @@ class Diaspora Logger::log("transmit: ".$logid."-".$guid." ".$dest_url); - if (!$queue_run && Queue::wasDelayed($contact["id"])) { - $return_code = 0; - } else { - if (!intval(Config::get("system", "diaspora_test"))) { - $content_type = (($public_batch) ? "application/magic-envelope+xml" : "application/json"); + if (!intval(Config::get("system", "diaspora_test"))) { + $content_type = (($public_batch) ? "application/magic-envelope+xml" : "application/json"); - $postResult = Network::post($dest_url."/", $envelope, ["Content-Type: ".$content_type]); - $return_code = $postResult->getReturnCode(); - } else { - Logger::log("test_mode"); - return 200; - } + $postResult = Network::post($dest_url."/", $envelope, ["Content-Type: ".$content_type]); + $return_code = $postResult->getReturnCode(); + } else { + Logger::log("test_mode"); + return 200; } Logger::log("transmit: ".$logid."-".$guid." to ".$dest_url." returns: ".$return_code); if (!$return_code || (($return_code == 503) && (stristr($postResult->getHeader(), "retry-after")))) { - if (!$no_queue && !empty($contact['contact-type']) && ($contact['contact-type'] != Contact::TYPE_RELAY)) { + if (!$no_defer && !empty($contact['contact-type']) && ($contact['contact-type'] != Contact::TYPE_RELAY)) { Logger::info('defer message', ['log' => $logid, 'guid' => $guid, 'destination' => $dest_url]); // defer message for redelivery Worker::defer(); @@ -3263,13 +3257,13 @@ class Diaspora * @param array $message The message data * @param bool $public_batch Is it a public post? * @param string $guid message guid - * @param bool $no_queue + * @param bool $no_defer Don't defer a failing delivery * * @return int Result of the transmission * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - private static function buildAndTransmit(array $owner, array $contact, $type, $message, $public_batch = false, $guid = "", $no_queue = false) + private static function buildAndTransmit(array $owner, array $contact, $type, $message, $public_batch = false, $guid = "", $no_defer = false) { $msg = self::buildPostXml($type, $message); @@ -3283,7 +3277,7 @@ class Diaspora $envelope = self::buildMessage($msg, $owner, $contact, $owner['uprvkey'], $contact['pubkey'], $public_batch); - $return_code = self::transmit($owner, $contact, $envelope, $public_batch, false, $guid, $no_queue); + $return_code = self::transmit($owner, $contact, $envelope, $public_batch, $guid, $no_defer); Logger::log("guid: ".$guid." result ".$return_code, Logger::DEBUG); diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 4e49dede8..7e3bc747c 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -47,9 +47,6 @@ class Cron // Fork the cron jobs in separate parts to avoid problems when one of them is crashing Hook::fork($a->queue['priority'], "cron"); - // run queue delivery process in the background - Worker::add(PRIORITY_NEGLIGIBLE, "Queue"); - // run the process to discover global contacts in the background Worker::add(PRIORITY_LOW, "DiscoverPoCo"); diff --git a/src/Worker/Queue.php b/src/Worker/Queue.php deleted file mode 100644 index 48e962f03..000000000 --- a/src/Worker/Queue.php +++ /dev/null @@ -1,164 +0,0 @@ - PRIORITY_LOW, 'dont_fork' => true], "Queue", (int) $q_item['id']); - } - } - Logger::log('filling queue jobs - end'); - return; - } - - - // delivering - $q_item = DBA::selectFirst('queue', [], ['id' => $queue_id]); - if (!DBA::isResult($q_item)) { - return; - } - - $contact = DBA::selectFirst('contact', [], ['id' => $q_item['cid']]); - if (!DBA::isResult($contact)) { - QueueModel::removeItem($q_item['id']); - return; - } - - if (empty($contact['notify']) || $contact['archive']) { - QueueModel::removeItem($q_item['id']); - return; - } - - $dead = Cache::get($cachekey_deadguy . $contact['notify']); - - if (!is_null($dead) && $dead && !$no_dead_check) { - Logger::log('queue: skipping known dead url: ' . $contact['notify']); - QueueModel::updateTime($q_item['id']); - return; - } - - if (!$no_dead_check) { - $server = PortableContact::detectServer($contact['url']); - - if ($server != "") { - $vital = Cache::get($cachekey_server . $server); - - if (is_null($vital)) { - Logger::log("Check server " . $server . " (" . $contact["network"] . ")"); - - $vital = PortableContact::checkServer($server, $contact["network"], true); - Cache::set($cachekey_server . $server, $vital, Cache::MINUTE); - } - - if (!is_null($vital) && !$vital) { - Logger::log('queue: skipping dead server: ' . $server); - QueueModel::updateTime($q_item['id']); - return; - } - } - } - - $user = DBA::selectFirst('user', [], ['uid' => $contact['uid']]); - if (!DBA::isResult($user)) { - QueueModel::removeItem($q_item['id']); - return; - } - - $data = $q_item['content']; - $public = $q_item['batch']; - $owner = User::getOwnerDataById($user['uid']); - - $deliver_status = 0; - - switch ($contact['network']) { - case Protocol::DFRN: - Logger::log('queue: dfrndelivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>'); - $deliver_status = DFRN::deliver($owner, $contact, $data); - - if (($deliver_status >= 200) && ($deliver_status <= 299)) { - QueueModel::removeItem($q_item['id']); - } else { - QueueModel::updateTime($q_item['id']); - Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE); - } - break; - - case Protocol::OSTATUS: - Logger::log('queue: slapdelivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>'); - $deliver_status = Salmon::slapper($owner, $contact['notify'], $data); - - if ($deliver_status == -1) { - QueueModel::updateTime($q_item['id']); - Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE); - } else { - QueueModel::removeItem($q_item['id']); - } - break; - - case Protocol::DIASPORA: - Logger::log('queue: diaspora_delivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>'); - $deliver_status = Diaspora::transmit($owner, $contact, $data, $public, true, 'Queue:' . $q_item['id'], true); - - if ((($deliver_status >= 200) && ($deliver_status <= 299)) || - ($contact['contact-type'] == Contact::TYPE_RELAY)) { - QueueModel::removeItem($q_item['id']); - } else { - QueueModel::updateTime($q_item['id']); - Cache::set($cachekey_deadguy . $contact['notify'], true, Cache::MINUTE); - } - break; - - default: - $params = ['owner' => $owner, 'contact' => $contact, 'queue' => $q_item, 'result' => false]; - Hook::callAll('queue_deliver', $params); - - if ($params['result']) { - QueueModel::removeItem($q_item['id']); - } else { - QueueModel::updateTime($q_item['id']); - } - break; - } - Logger::log('Deliver status ' . (int)$deliver_status . ' for item ' . $q_item['id'] . ' to ' . $contact['name'] . ' <' . $contact['url'] . '>'); - - return; - } -} diff --git a/update.php b/update.php index e619ec89d..40a3e8ff0 100644 --- a/update.php +++ b/update.php @@ -13,6 +13,7 @@ use Friendica\Model\GContact; use Friendica\Model\Item; use Friendica\Model\User; use Friendica\Util\DateTimeFormat; +use Friendica\Worker\Delivery; /** * @@ -334,7 +335,7 @@ function update_1298() DBA::update('profile', [$translateKey => $key], ['id' => $data['id']]); Logger::notice('Updated contact', ['action' => 'update', 'contact' => $data['id'], "$translateKey" => $key, 'was' => $data[$translateKey]]); - Worker::add(PRIORITY_LOW, 'ProfileUpdate', $data['id']); + Worker::add(PRIORITY_LOW, 'ProfileUpdate', $data['id']); Contact::updateSelfFromUserID($data['id']); GContact::updateForUser($data['id']); $success++; @@ -346,3 +347,31 @@ function update_1298() } return Update::SUCCESS; } + +function update_1309() +{ + $queue = DBA::select('queue', ['id', 'cid', 'guid']); + while ($entry = DBA::fetch($queue)) { + $contact = DBA::selectFirst('contact', ['uid'], ['id' => $entry['cid']]); + if (!DBA::isResult($contact)) { + continue; + } + + $item = Item::selectFirst(['id', 'gravity'], ['uid' => $contact['uid'], 'guid' => $entry['guid']]); + if (!DBA::isResult($item)) { + continue; + } + + if ($item['gravity'] == GRAVITY_PARENT) { + $cmd = Delivery::POST; + } else { + $cmd = Delivery::COMMENT; + } + + $deliver_options = ['priority' => PRIORITY_MEDIUM, 'dont_fork' => true]; + Worker::add($deliver_options, 'Delivery', $cmd, $item['id'], $entry['cid']); + Logger::info('Added delivery worker', ['command' => $cmd, 'item' => $item['id'], 'contact' => $entry['cid']]); + DBA::delete('queue', ['id' => $entry['id']]); + } + return Update::SUCCESS; +} diff --git a/view/templates/admin/summary.tpl b/view/templates/admin/summary.tpl index 07c2957bd..09419929d 100644 --- a/view/templates/admin/summary.tpl +++ b/view/templates/admin/summary.tpl @@ -11,7 +11,7 @@
{{$queues.label}}
-
{{$queues.queue}} - {{$queues.deferred}} - {{$queues.workerq}}
+
{{$queues.deferred}} - {{$queues.workerq}}
{{$pending.0}}
diff --git a/view/theme/frio/templates/admin/summary.tpl b/view/theme/frio/templates/admin/summary.tpl index 502da3187..d183e3fa2 100644 --- a/view/theme/frio/templates/admin/summary.tpl +++ b/view/theme/frio/templates/admin/summary.tpl @@ -14,7 +14,7 @@ {{* The work queues short statistic. *}}
{{$queues.label}}
-
{{$queues.queue}} - {{$queues.deferred}} - {{$queues.workerq}}
+
{{$queues.deferred}} - {{$queues.workerq}}
{{* Number of pending registrations. *}}