From 0093f863fd584cc9024a6aad3c23325687b5e9a6 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Feb 2018 11:26:24 +0000 Subject: [PATCH 1/3] Don't add already queued items from Diaspora to the queue again --- boot.php | 2 +- database.sql | 3 ++- src/Database/DBStructure.php | 1 + src/Model/Queue.php | 8 +++++--- src/Protocol/Diaspora.php | 26 ++++++++------------------ src/Worker/Delivery.php | 2 +- src/Worker/Queue.php | 2 +- 7 files changed, 19 insertions(+), 25 deletions(-) diff --git a/boot.php b/boot.php index b2d9f01927..48c9b29444 100644 --- a/boot.php +++ b/boot.php @@ -39,7 +39,7 @@ define('FRIENDICA_PLATFORM', 'Friendica'); define('FRIENDICA_CODENAME', 'Asparagus'); define('FRIENDICA_VERSION', '3.6-dev'); define('DFRN_PROTOCOL_VERSION', '2.23'); -define('DB_UPDATE_VERSION', 1249); +define('DB_UPDATE_VERSION', 1250); define('NEW_UPDATE_ROUTINE_VERSION', 1170); /** diff --git a/database.sql b/database.sql index 85cd811c1b..7193b59d17 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 3.6-dev (Asparagus) --- DB_UPDATE_VERSION 1248 +-- DB_UPDATE_VERSION 1250 -- ------------------------------------------ @@ -855,6 +855,7 @@ CREATE TABLE IF NOT EXISTS `queue` ( `id` int NOT NULL auto_increment COMMENT '', `cid` int NOT NULL DEFAULT 0 COMMENT '', `network` varchar(32) NOT NULL DEFAULT '' COMMENT '', + `guid` varchar(255) NOT NULL DEFAULT '' COMMENT '', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `content` mediumtext COMMENT '', diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php index 4c05782f17..4cf96db260 100644 --- a/src/Database/DBStructure.php +++ b/src/Database/DBStructure.php @@ -1551,6 +1551,7 @@ class DBStructure "id" => ["type" => "int", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => ""], "cid" => ["type" => "int", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => ""], "network" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""], + "guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""], "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""], "content" => ["type" => "mediumtext", "comment" => ""], diff --git a/src/Model/Queue.php b/src/Model/Queue.php index 3c3cf379a4..2d7873a1e0 100644 --- a/src/Model/Queue.php +++ b/src/Model/Queue.php @@ -66,7 +66,7 @@ class Queue * @param string $msg message * @param boolean $batch batch, default false */ - public static function add($cid, $network, $msg, $batch = false) + public static function add($cid, $network, $msg, $batch = false, $guid = '') { $max_queue = Config::get('system', 'max_contact_queue'); @@ -86,10 +86,10 @@ class Queue if (DBM::is_result($r)) { if ($batch && ($r[0]['total'] > $batch_queue)) { - logger('add_to_queue: too many queued items for batch server ' . $cid . ' - discarding message'); + logger('too many queued items for batch server ' . $cid . ' - discarding message'); return; } elseif ((! $batch) && ($r[0]['total'] > $max_queue)) { - logger('add_to_queue: too many queued items for contact ' . $cid . ' - discarding message'); + logger('too many queued items for contact ' . $cid . ' - discarding message'); return; } } @@ -97,10 +97,12 @@ class Queue dba::insert('queue', [ 'cid' => $cid, 'network' => $network, + 'guid' => $guid, 'created' => DateTimeFormat::utcNow(), 'last' => DateTimeFormat::utcNow(), 'content' => $msg, 'batch' =>($batch) ? 1 : 0 ]); + logger('Added item ' . $guid . ' for ' . $cid); } } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 0f5c9c0c9d..534f6ed6ba 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -3181,7 +3181,7 @@ class Diaspora * * @return int Result of the transmission */ - public static function transmit($owner, $contact, $envelope, $public_batch, $queue_run = false, $guid = "") + public static function transmit($owner, $contact, $envelope, $public_batch, $queue_run = false, $guid = "", $no_queue = false) { $a = get_app(); @@ -3224,24 +3224,14 @@ class Diaspora logger("transmit: ".$logid."-".$guid." returns: ".$return_code); if (!$return_code || (($return_code == 503) && (stristr($a->get_curl_headers(), "retry-after")))) { - logger("queue message"); - - $r = q( - "SELECT `id` FROM `queue` WHERE `cid` = %d AND `network` = '%s' AND `content` = '%s' AND `batch` = %d LIMIT 1", - intval($contact["id"]), - dbesc(NETWORK_DIASPORA), - dbesc($envelope), - intval($public_batch) - ); - if ($r) { - logger("add_to_queue ignored - identical item already in queue"); - } else { + if (!$no_queue) { + logger("queue message"); // queue message for redelivery - Queue::add($contact["id"], NETWORK_DIASPORA, $envelope, $public_batch); - - // The message could not be delivered. We mark the contact as "dead" - Contact::markForArchival($contact); + Queue::add($contact["id"], NETWORK_DIASPORA, $envelope, $public_batch, $guid); } + + // The message could not be delivered. We mark the contact as "dead" + Contact::markForArchival($contact); } elseif (($return_code >= 200) && ($return_code <= 299)) { // We successfully delivered a message, the contact is alive Contact::unmarkForArchival($contact); @@ -3294,7 +3284,7 @@ class Diaspora $envelope = self::buildMessage($msg, $owner, $contact, $owner['uprvkey'], $contact['pubkey'], $public_batch); if ($spool) { - Queue::add($contact['id'], NETWORK_DIASPORA, $envelope, $public_batch); + Queue::add($contact['id'], NETWORK_DIASPORA, $envelope, $public_batch, $guid); return true; } else { $return_code = self::transmit($owner, $contact, $envelope, $public_batch, false, $guid); diff --git a/src/Worker/Delivery.php b/src/Worker/Delivery.php index 0a77f7b5be..2b9a00176e 100644 --- a/src/Worker/Delivery.php +++ b/src/Worker/Delivery.php @@ -322,7 +322,7 @@ class Delivery { if ($deliver_status < 0) { logger('notifier: delivery failed: queuing message'); - Queue::add($contact['id'], NETWORK_DFRN, $atom); + Queue::add($contact['id'], NETWORK_DFRN, $atom, false, $target_item['guid']); // The message could not be delivered. We mark the contact as "dead" Contact::markForArchival($contact); diff --git a/src/Worker/Queue.php b/src/Worker/Queue.php index 10a83e5c4b..f44be4918c 100644 --- a/src/Worker/Queue.php +++ b/src/Worker/Queue.php @@ -145,7 +145,7 @@ class Queue case NETWORK_DIASPORA: if ($contact['notify']) { logger('queue: diaspora_delivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>'); - $deliver_status = Diaspora::transmit($owner, $contact, $data, $public, true); + $deliver_status = Diaspora::transmit($owner, $contact, $data, $public, true, 'Queue:' . $q_item['id'], true); if ($deliver_status == (-1)) { QueueModel::updateTime($q_item['id']); From 8ea572ebd19083f474a698336af880c84a1380e8 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Feb 2018 13:09:09 +0000 Subject: [PATCH 2/3] Added a config value to ignore server checks --- doc/htconfig.md | 1 + src/Worker/Queue.php | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/doc/htconfig.md b/doc/htconfig.md index e09c37b74d..3a482742d2 100644 --- a/doc/htconfig.md +++ b/doc/htconfig.md @@ -69,6 +69,7 @@ Example: To set the automatic database cleanup process add this line to your .ht * **ostatus_poll_timeframe** - Defines how old an item can be to try to complete the conversation with it. * **paranoia** (Boolean) - Log out users if their IP address changed. * **permit_crawling** (Boolean) - Restricts the search for not logged in users to one search per minute. +* **queue_no_dead_check** (Boolean) - Ignore if the target contact or server seems to be dead during queue delivery. * **worker_debug** (Boolean) - If enabled, it prints out the number of running processes split by priority. * **worker_fetch_limit** - Number of worker tasks that are fetched in a single query. Default is 1. * **profiler** (Boolean) - Enable internal timings to help optimize code. Needed for "rendertime" addon. Default is false. diff --git a/src/Worker/Queue.php b/src/Worker/Queue.php index f44be4918c..d92be19fe3 100644 --- a/src/Worker/Queue.php +++ b/src/Worker/Queue.php @@ -6,6 +6,7 @@ namespace Friendica\Worker; use Friendica\Core\Addon; use Friendica\Core\Cache; +use Friendica\Core\Config; use Friendica\Core\Worker; use Friendica\Database\DBM; use Friendica\Model\Queue as QueueModel; @@ -28,6 +29,8 @@ class Queue $cachekey_deadguy = 'queue_run:deadguy:'; $cachekey_server = 'queue_run:server:'; + $no_dead_check = Config::get('system', 'queue_no_dead_check', false); + if (!$queue_id) { logger('queue: start'); @@ -80,28 +83,30 @@ class Queue $dead = Cache::get($cachekey_deadguy . $contact['notify']); - if (!is_null($dead) && $dead) { + if (!is_null($dead) && $dead && !$no_dead_check) { logger('queue: skipping known dead url: ' . $contact['notify']); QueueModel::updateTime($q_item['id']); return; } - $server = PortableContact::detectServer($contact['url']); + if (!$no_dead_check) { + $server = PortableContact::detectServer($contact['url']); - if ($server != "") { - $vital = Cache::get($cachekey_server . $server); + if ($server != "") { + $vital = Cache::get($cachekey_server . $server); - if (is_null($vital)) { - logger("Check server " . $server . " (" . $contact["network"] . ")"); + if (is_null($vital)) { + logger("Check server " . $server . " (" . $contact["network"] . ")"); - $vital = PortableContact::checkServer($server, $contact["network"], true); - Cache::set($cachekey_server . $server, $vital, CACHE_QUARTER_HOUR); - } + $vital = PortableContact::checkServer($server, $contact["network"], true); + Cache::set($cachekey_server . $server, $vital, CACHE_QUARTER_HOUR); + } - if (!is_null($vital) && !$vital) { - logger('queue: skipping dead server: ' . $server); - QueueModel::updateTime($q_item['id']); - return; + if (!is_null($vital) && !$vital) { + logger('queue: skipping dead server: ' . $server); + QueueModel::updateTime($q_item['id']); + return; + } } } From ac7b0661d25e0be0e896b60167122c32ad163ef5 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Feb 2018 15:03:18 +0000 Subject: [PATCH 3/3] Added documentation --- src/Database/DBStructure.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php index 4cf96db260..60859e53c3 100644 --- a/src/Database/DBStructure.php +++ b/src/Database/DBStructure.php @@ -1549,10 +1549,10 @@ class DBStructure "comment" => "Queue for messages that couldn't be delivered", "fields" => [ "id" => ["type" => "int", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => ""], - "cid" => ["type" => "int", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => ""], - "network" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""], - "guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""], + "cid" => ["type" => "int", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "Message receiver"], + "network" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Receiver's network"], + "guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Unique GUID of the message"], + "created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date, when the message was created"], "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""], "content" => ["type" => "mediumtext", "comment" => ""], "batch" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],