From e21db0fe6d29f6f752415d8b535206000053734b Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 31 Dec 2022 23:42:00 +0000 Subject: [PATCH 1/8] The inbox-status can now be archived for a whole server --- database.sql | 7 ++-- doc/database/db_inbox-status.md | 3 ++ src/Database/PostUpdate.php | 60 ++++++++++++++++++++++++++++++++- src/Model/APContact.php | 22 +++++++----- src/Model/GServer.php | 14 +++++++- src/Util/HTTPSignature.php | 10 +++++- static/dbstructure.config.php | 4 ++- 7 files changed, 105 insertions(+), 15 deletions(-) diff --git a/database.sql b/database.sql index 2d5a1dbaa..19007e6c0 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2023.03-dev (Giant Rhubarb) --- DB_UPDATE_VERSION 1506 +-- DB_UPDATE_VERSION 1507 -- ------------------------------------------ @@ -824,6 +824,7 @@ CREATE TABLE IF NOT EXISTS `inbox-entry-receiver` ( CREATE TABLE IF NOT EXISTS `inbox-status` ( `url` varbinary(383) NOT NULL COMMENT 'URL of the inbox', `uri-id` int unsigned COMMENT 'Item-uri id of inbox url', + `gsid` int unsigned COMMENT 'ID of the related server', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation date of this entry', `success` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last successful delivery', `failure` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed delivery', @@ -832,7 +833,9 @@ CREATE TABLE IF NOT EXISTS `inbox-status` ( `shared` boolean NOT NULL DEFAULT '0' COMMENT 'Is it a shared inbox?', PRIMARY KEY(`url`), INDEX `uri-id` (`uri-id`), - FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE + INDEX `gsid` (`gsid`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Status of ActivityPub inboxes'; -- diff --git a/doc/database/db_inbox-status.md b/doc/database/db_inbox-status.md index abcb33da4..ccb28e9bb 100644 --- a/doc/database/db_inbox-status.md +++ b/doc/database/db_inbox-status.md @@ -10,6 +10,7 @@ Fields | -------- | ------------------------------------ | -------------- | ---- | --- | ------------------- | ----- | | url | URL of the inbox | varbinary(383) | NO | PRI | NULL | | | uri-id | Item-uri id of inbox url | int unsigned | YES | | NULL | | +| gsid | ID of the related server | int unsigned | YES | | NULL | | | created | Creation date of this entry | datetime | NO | | 0001-01-01 00:00:00 | | | success | Date of the last successful delivery | datetime | NO | | 0001-01-01 00:00:00 | | | failure | Date of the last failed delivery | datetime | NO | | 0001-01-01 00:00:00 | | @@ -24,6 +25,7 @@ Indexes | ------- | ------ | | PRIMARY | url | | uri-id | uri-id | +| gsid | gsid | Foreign Keys ------------ @@ -31,5 +33,6 @@ Foreign Keys | Field | Target Table | Target Field | |-------|--------------|--------------| | uri-id | [item-uri](help/database/db_item-uri) | id | +| gsid | [gserver](help/database/db_gserver) | id | Return to [database documentation](help/database) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index d54c3f221..cfb6b1569 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -50,7 +50,7 @@ class PostUpdate // Needed for the helper function to read from the legacy term table const OBJECT_TYPE_POST = 1; - const VERSION = 1484; + const VERSION = 1507; /** * Calls the post update functions @@ -117,6 +117,9 @@ class PostUpdate if (!self::update1484()) { return false; } + if (!self::update1507()) { + return false; + } return true; } @@ -1184,4 +1187,59 @@ class PostUpdate return false; } + + /** + * update the "gsid" (global server id) field in the inbox-status table + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws \ImagickException + */ + private static function update1507() + { + // Was the script completed? + if (DI::keyValue()->get('post_update_version') >= 1507) { + return true; + } + + $id = DI::keyValue()->get('post_update_version_1507_id') ?? ''; + + Logger::info('Start', ['apcontact' => $id]); + + $start_id = $id; + $rows = 0; + $condition = ["`url` > ? AND NOT `gsid` IS NULL", $id]; + $params = ['order' => ['url'], 'limit' => 10000]; + $apcontacts = DBA::select('apcontact', ['url', 'gsid', 'sharedinbox', 'inbox'], $condition, $params); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($apcontact = DBA::fetch($apcontacts)) { + $id = $apcontact['url']; + + $inbox = [$apcontact['inbox']]; + if (!empty($apcontact['sharedinbox'])) { + $inbox[] = $apcontact['sharedinbox']; + } + $condition = DBA::mergeConditions(['url' => $inbox], ["`gsid` IS NULL"]); + DBA::update('inbox-status', ['gsid' => $apcontact['gsid']], $condition); + ++$rows; + } + DBA::close($apcontacts); + + DI::keyValue()->set('post_update_version_1507_id', $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + if ($start_id == $id) { + DI::keyValue()->set('post_update_version', 1507); + Logger::info('Done'); + return true; + } + + return false; + } } diff --git a/src/Model/APContact.php b/src/Model/APContact.php index fd748f1cd..851916de9 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -226,14 +226,11 @@ class APContact $apcontact['following'] = JsonLD::fetchElement($compacted, 'as:following', '@id'); $apcontact['followers'] = JsonLD::fetchElement($compacted, 'as:followers', '@id'); $apcontact['inbox'] = (JsonLD::fetchElement($compacted, 'ldp:inbox', '@id') ?? ''); - self::unarchiveInbox($apcontact['inbox'], false); - $apcontact['outbox'] = JsonLD::fetchElement($compacted, 'as:outbox', '@id'); $apcontact['sharedinbox'] = ''; if (!empty($compacted['as:endpoints'])) { $apcontact['sharedinbox'] = (JsonLD::fetchElement($compacted['as:endpoints'], 'as:sharedInbox', '@id') ?? ''); - self::unarchiveInbox($apcontact['sharedinbox'], true); } $apcontact['featured'] = JsonLD::fetchElement($compacted, 'toot:featured', '@id'); @@ -427,6 +424,12 @@ class APContact $apcontact['gsid'] = null; } + self::unarchiveInbox($apcontact['inbox'], false, $apcontact['gsid']); + + if (!empty($apcontact['sharedinbox'])) { + self::unarchiveInbox($apcontact['sharedinbox'], true, $apcontact['gsid']); + } + if ($apcontact['url'] == $apcontact['alias']) { $apcontact['alias'] = null; } @@ -517,7 +520,7 @@ class APContact { if (!empty($apcontact['inbox'])) { Logger::info('Set inbox status to failure', ['inbox' => $apcontact['inbox']]); - HTTPSignature::setInboxStatus($apcontact['inbox'], false); + HTTPSignature::setInboxStatus($apcontact['inbox'], false, false, $apcontact['gsid']); } if (!empty($apcontact['sharedinbox'])) { @@ -527,7 +530,7 @@ class APContact if (!$available) { // If all known personal inboxes are failing then set their shared inbox to failure as well Logger::info('Set shared inbox status to failure', ['sharedinbox' => $apcontact['sharedinbox']]); - HTTPSignature::setInboxStatus($apcontact['sharedinbox'], false, true); + HTTPSignature::setInboxStatus($apcontact['sharedinbox'], false, true, $apcontact['gsid']); } } } @@ -542,11 +545,11 @@ class APContact { if (!empty($apcontact['inbox'])) { Logger::info('Set inbox status to success', ['inbox' => $apcontact['inbox']]); - HTTPSignature::setInboxStatus($apcontact['inbox'], true); + HTTPSignature::setInboxStatus($apcontact['inbox'], true, false, $apcontact['gsid']); } if (!empty($apcontact['sharedinbox'])) { Logger::info('Set shared inbox status to success', ['sharedinbox' => $apcontact['sharedinbox']]); - HTTPSignature::setInboxStatus($apcontact['sharedinbox'], true, true); + HTTPSignature::setInboxStatus($apcontact['sharedinbox'], true, true, $apcontact['gsid']); } } @@ -555,15 +558,16 @@ class APContact * * @param string $url inbox url * @param boolean $shared Shared Inbox + * @param int $gsid Global server id * @return void */ - private static function unarchiveInbox(string $url, bool $shared) + private static function unarchiveInbox(string $url, bool $shared, int $gsid = null) { if (empty($url)) { return; } - HTTPSignature::setInboxStatus($url, true, $shared); + HTTPSignature::setInboxStatus($url, true, $shared, $gsid); } /** diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 477b1f323..86fb78f79 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -408,7 +408,7 @@ class GServer ['nurl' => Strings::normaliseLink($url)]); Logger::info('Set failed status for existing server', ['url' => $url]); if (self::isDefunct($gserver)) { - Contact::update(['archive' => true], ['gsid' => $gserver['id']]); + self::archiveContacts($gserver['id']); } return; } @@ -418,6 +418,18 @@ class GServer Logger::info('Set failed status for new server', ['url' => $url]); } + /** + * Archive server related contacts and inboxes + * + * @param integer $gsid + * @return void + */ + private static function archiveContacts(int $gsid) + { + Contact::update(['archive' => true], ['gsid' => $gsid]); + DBA::update('inbox-status', ['archive' => true], ['gsid' => $gsid]); + } + /** * Remove unwanted content from the given URL * diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index 84219b311..0543cbcdf 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -332,15 +332,19 @@ class HTTPSignature * @param string $url The URL of the inbox * @param boolean $success Transmission status * @param boolean $shared The inbox is a shared inbox + * @param int $gsid Server ID * @throws \Exception */ - static public function setInboxStatus(string $url, bool $success, bool $shared = false) + static public function setInboxStatus(string $url, bool $success, bool $shared = false, int $gsid = null) { $now = DateTimeFormat::utcNow(); $status = DBA::selectFirst('inbox-status', [], ['url' => $url]); if (!DBA::isResult($status)) { $insertFields = ['url' => $url, 'uri-id' => ItemURI::getIdByURI($url), 'created' => $now, 'shared' => $shared]; + if (!empty($gsid)) { + $insertFields['gsid'] = $gsid; + } if (!DBA::insert('inbox-status', $insertFields, Database::INSERT_IGNORE)) { Logger::warning('Unable to insert inbox-status row', $insertFields); return; @@ -355,6 +359,10 @@ class HTTPSignature $fields = ['failure' => $now]; } + if (!empty($gsid)) { + $fields['gsid'] = $gsid; + } + if ($status['failure'] > DBA::NULL_DATETIME) { $new_previous_stamp = strtotime($status['failure']); $old_previous_stamp = strtotime($status['previous']); diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index cf6131c2e..7736dafd4 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1506); + define('DB_UPDATE_VERSION', 1507); } return [ @@ -872,6 +872,7 @@ return [ "fields" => [ "url" => ["type" => "varbinary(383)", "not null" => "1", "primary" => "1", "comment" => "URL of the inbox"], "uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"], + "gsid" => ["type" => "int unsigned", "foreign" => ["gserver" => "id", "on delete" => "restrict"], "comment" => "ID of the related server"], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation date of this entry"], "success" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last successful delivery"], "failure" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last failed delivery"], @@ -882,6 +883,7 @@ return [ "indexes" => [ "PRIMARY" => ["url"], "uri-id" => ["uri-id"], + "gsid" => ["gsid"], ] ], "intro" => [ From 12d37835b4f9c3691460b7386566d39de14b2c4e Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 Jan 2023 00:18:01 +0000 Subject: [PATCH 2/8] Set "archive" status --- src/Database/PostUpdate.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index cfb6b1569..41333a34b 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -1225,7 +1225,7 @@ class PostUpdate $inbox[] = $apcontact['sharedinbox']; } $condition = DBA::mergeConditions(['url' => $inbox], ["`gsid` IS NULL"]); - DBA::update('inbox-status', ['gsid' => $apcontact['gsid']], $condition); + DBA::update('inbox-status', ['gsid' => $apcontact['gsid'], 'archive' => GServer::isDefunctById($apcontact['gsid'])], $condition); ++$rows; } DBA::close($apcontacts); From 06403f1546535ed0cc3d1bc3877977fda438f693 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 Jan 2023 07:52:28 +0000 Subject: [PATCH 3/8] License updated --- src/Database/PostUpdate.php | 5 +++-- src/Model/APContact.php | 2 +- src/Model/GServer.php | 2 +- src/Util/HTTPSignature.php | 2 +- static/dbstructure.config.php | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 41333a34b..9feee2bcc 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -1,6 +1,6 @@ $inbox], ["`gsid` IS NULL"]); +// $condition = DBA::mergeConditions(['url' => $inbox], ["`gsid` IS NULL"]); + $condition = ['url' => $inbox]; DBA::update('inbox-status', ['gsid' => $apcontact['gsid'], 'archive' => GServer::isDefunctById($apcontact['gsid'])], $condition); ++$rows; } diff --git a/src/Model/APContact.php b/src/Model/APContact.php index 851916de9..09a6cb9e7 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -1,6 +1,6 @@ Date: Sun, 1 Jan 2023 10:52:22 +0000 Subject: [PATCH 4/8] Add the server id for all Diaspora contacts --- src/Database/PostUpdate.php | 64 +++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 9feee2bcc..acc98c882 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -38,6 +38,7 @@ use Friendica\Protocol\ActivityPub\Processor; use Friendica\Protocol\ActivityPub\Receiver; use Friendica\Util\JsonLD; use Friendica\Util\Strings; +use GuzzleHttp\Psr7\Uri; /** * These database-intensive post update routines are meant to be executed in the background by the cronjob. @@ -117,6 +118,9 @@ class PostUpdate if (!self::update1484()) { return false; } + if (!self::update1506()) { + return false; + } if (!self::update1507()) { return false; } @@ -1188,6 +1192,63 @@ class PostUpdate return false; } + /** + * update the "gsid" (global server id) field in the contact table + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws \ImagickException + */ + private static function update1506() + { + // Was the script completed? + if (DI::keyValue()->get('post_update_version') >= 1506) { + return true; + } + + $id = DI::keyValue()->get('post_update_version_1506_id') ?? 0; + + Logger::info('Start', ['contact' => $id]); + + $start_id = $id; + $rows = 0; + $condition = ["`id` > ? AND `gsid` IS NULL AND `network` = ?", $id, Protocol::DIASPORA]; + $params = ['order' => ['id'], 'limit' => 10000]; + $contacts = DBA::select('contact', ['id', 'url'], $condition, $params); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($contact = DBA::fetch($contacts)) { + $id = $contact['id']; + + $parts = parse_url($contact['url']); + unset($parts['path']); + $server = (string)Uri::fromParts($parts); + + DBA::update('contact', + ['gsid' => GServer::getID($server, true), 'baseurl' => GServer::cleanURL($server)], + ['id' => $contact['id']]); + + ++$rows; + } + DBA::close($contacts); + + DI::keyValue()->set('post_update_version_1506_id', $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + if ($start_id == $id) { + DI::keyValue()->set('post_update_version', 1506); + Logger::info('Done'); + return true; + } + + return false; + } + /** * update the "gsid" (global server id) field in the inbox-status table * @@ -1224,8 +1285,7 @@ class PostUpdate if (!empty($apcontact['sharedinbox'])) { $inbox[] = $apcontact['sharedinbox']; } -// $condition = DBA::mergeConditions(['url' => $inbox], ["`gsid` IS NULL"]); - $condition = ['url' => $inbox]; + $condition = DBA::mergeConditions(['url' => $inbox], ["`gsid` IS NULL"]); DBA::update('inbox-status', ['gsid' => $apcontact['gsid'], 'archive' => GServer::isDefunctById($apcontact['gsid'])], $condition); ++$rows; } From 7307646cf815fb9b0082827e79f8fc01326489b1 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 Jan 2023 15:13:44 +0000 Subject: [PATCH 5/8] Use server id on direct delivery --- src/Core/Worker/Cron.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Core/Worker/Cron.php b/src/Core/Worker/Cron.php index 8e354344b..bc6afc32e 100644 --- a/src/Core/Worker/Cron.php +++ b/src/Core/Worker/Cron.php @@ -164,16 +164,18 @@ class Cron */ private static function deliverAPPosts() { - $deliveries = DBA::p("SELECT `item-uri`.`uri` AS `inbox`, MAX(`failed`) AS `failed` FROM `post-delivery` INNER JOIN `item-uri` ON `item-uri`.`id` = `post-delivery`.`inbox-id` GROUP BY `inbox` ORDER BY RAND()"); + $deliveries = DBA::p("SELECT `item-uri`.`uri` AS `inbox`, MAX(`gsid`) AS `gsid`, MAX(`failed`) AS `failed` FROM `post-delivery` INNER JOIN `item-uri` ON `item-uri`.`id` = `post-delivery`.`inbox-id` LEFT JOIN `inbox-status` ON `inbox-status`.`url` = `item-uri`.`uri` GROUP BY `inbox` ORDER BY RAND()"); while ($delivery = DBA::fetch($deliveries)) { if ($delivery['failed'] > 0) { Logger::info('Removing failed deliveries', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed']]); Post\Delivery::removeFailed($delivery['inbox']); } - - if ($delivery['failed'] == 0) { + if (($delivery['failed'] == 0) && !empty($delivery['gsid']) && GServer::isReachableById($delivery['gsid'])) { $result = ActivityPub\Delivery::deliver($delivery['inbox']); Logger::info('Directly deliver inbox', ['inbox' => $delivery['inbox'], 'result' => $result['success']]); + if (!$result['success']) { + GServer::setFailureById($delivery['gsid']); + } continue; } elseif ($delivery['failed'] < 3) { $priority = Worker::PRIORITY_HIGH; From fe897c7ef4a99d68761ddcb49b6c0fa9944847f1 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 Jan 2023 15:14:45 +0000 Subject: [PATCH 6/8] Increase the year --- src/Core/Worker/Cron.php | 2 +- src/Worker/Notifier.php | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Core/Worker/Cron.php b/src/Core/Worker/Cron.php index bc6afc32e..d45592f37 100644 --- a/src/Core/Worker/Cron.php +++ b/src/Core/Worker/Cron.php @@ -1,6 +1,6 @@ $target_item['id'], 'url' => $target_item['uri'], 'verb' => $target_item['verb']]); + return ['count' => 0, 'contacts' => []]; + } $delivery_queue_count = 0; $contacts = []; From df5305f384bdf2ca02d5ae183e39466ee0b8298b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 Jan 2023 15:15:16 +0000 Subject: [PATCH 7/8] Increase another forgotten year --- src/Worker/Notifier.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index 1cb1cad92..c2f542dd3 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -1,6 +1,6 @@ Date: Sun, 1 Jan 2023 15:37:47 +0000 Subject: [PATCH 8/8] Close the database --- src/Core/Worker/Cron.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Core/Worker/Cron.php b/src/Core/Worker/Cron.php index d45592f37..e358ccd85 100644 --- a/src/Core/Worker/Cron.php +++ b/src/Core/Worker/Cron.php @@ -192,6 +192,8 @@ class Cron } } + DBA::close($deliveries); + // Optimizing this table only last seconds if (DI::config()->get('system', 'optimize_tables')) { Logger::info('Optimize start');