Merge pull request #12587 from annando/inbox-gsid

The inbox-status can now be archived for a whole server

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2023.03-dev (Giant Rhubarb)
--- DB_UPDATE_VERSION 1506
+-- DB_UPDATE_VERSION 1507
 -- ------------------------------------------
 
 
@@ -824,6 +824,7 @@ CREATE TABLE IF NOT EXISTS `inbox-entry-receiver` (
 CREATE TABLE IF NOT EXISTS `inbox-status` (
 	`url` varbinary(383) NOT NULL COMMENT 'URL of the inbox',
 	`uri-id` int unsigned COMMENT 'Item-uri id of inbox url',
+	`gsid` int unsigned COMMENT 'ID of the related server',
 	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation date of this entry',
 	`success` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last successful delivery',
 	`failure` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed delivery',
@@ -832,7 +833,9 @@ CREATE TABLE IF NOT EXISTS `inbox-status` (
 	`shared` boolean NOT NULL DEFAULT '0' COMMENT 'Is it a shared inbox?',
 	 PRIMARY KEY(`url`),
 	 INDEX `uri-id` (`uri-id`),
-	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+	 INDEX `gsid` (`gsid`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Status of ActivityPub inboxes';
 
 --

doc/database/db_inbox-status.md

@@ -10,6 +10,7 @@ Fields
 | -------- | ------------------------------------ | -------------- | ---- | --- | ------------------- | ----- |
 | url      | URL of the inbox                     | varbinary(383) | NO   | PRI | NULL                |       |
 | uri-id   | Item-uri id of inbox url             | int unsigned   | YES  |     | NULL                |       |
+| gsid     | ID of the related server             | int unsigned   | YES  |     | NULL                |       |
 | created  | Creation date of this entry          | datetime       | NO   |     | 0001-01-01 00:00:00 |       |
 | success  | Date of the last successful delivery | datetime       | NO   |     | 0001-01-01 00:00:00 |       |
 | failure  | Date of the last failed delivery     | datetime       | NO   |     | 0001-01-01 00:00:00 |       |
@@ -24,6 +25,7 @@ Indexes
 | ------- | ------ |
 | PRIMARY | url    |
 | uri-id  | uri-id |
+| gsid    | gsid   |
 
 Foreign Keys
 ------------
@@ -31,5 +33,6 @@ Foreign Keys
 | Field | Target Table | Target Field |
 |-------|--------------|--------------|
 | uri-id | [item-uri](help/database/db_item-uri) | id |
+| gsid | [gserver](help/database/db_gserver) | id |
 
 Return to [database documentation](help/database)

src/Core/Worker/Cron.php

@@ -164,16 +164,18 @@ class Cron
 	 */
 	private static function deliverAPPosts()
 	{
-		$deliveries = DBA::p("SELECT `item-uri`.`uri` AS `inbox`, MAX(`failed`) AS `failed` FROM `post-delivery` INNER JOIN `item-uri` ON `item-uri`.`id` = `post-delivery`.`inbox-id` GROUP BY `inbox` ORDER BY RAND()");
+		$deliveries = DBA::p("SELECT `item-uri`.`uri` AS `inbox`, MAX(`gsid`) AS `gsid`, MAX(`failed`) AS `failed` FROM `post-delivery` INNER JOIN `item-uri` ON `item-uri`.`id` = `post-delivery`.`inbox-id` LEFT JOIN `inbox-status` ON `inbox-status`.`url` = `item-uri`.`uri` GROUP BY `inbox` ORDER BY RAND()");
 		while ($delivery = DBA::fetch($deliveries)) {
 			if ($delivery['failed'] > 0) {
 				Logger::info('Removing failed deliveries', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed']]);
 				Post\Delivery::removeFailed($delivery['inbox']);
 			}
-
+			if (($delivery['failed'] == 0) && !empty($delivery['gsid']) && GServer::isReachableById($delivery['gsid'])) {
-			if ($delivery['failed'] == 0) {
 				$result = ActivityPub\Delivery::deliver($delivery['inbox']);
 				Logger::info('Directly deliver inbox', ['inbox' => $delivery['inbox'], 'result' => $result['success']]);
+				if (!$result['success']) {
+					GServer::setFailureById($delivery['gsid']);
+				}
 				continue;
 			} elseif ($delivery['failed'] < 3) {
 				$priority = Worker::PRIORITY_HIGH;
@@ -190,6 +192,8 @@ class Cron
 			}
 		}
 
+		DBA::close($deliveries);
+
 		// Optimizing this table only last seconds
 		if (DI::config()->get('system', 'optimize_tables')) {
 			Logger::info('Optimize start');

src/Database/PostUpdate.php

@@ -38,6 +38,7 @@ use Friendica\Protocol\ActivityPub\Processor;
 use Friendica\Protocol\ActivityPub\Receiver;
 use Friendica\Util\JsonLD;
 use Friendica\Util\Strings;
+use GuzzleHttp\Psr7\Uri;
 
 /**
  * These database-intensive post update routines are meant to be executed in the background by the cronjob.
@@ -50,7 +51,7 @@ class PostUpdate
 	// Needed for the helper function to read from the legacy term table
 	const OBJECT_TYPE_POST  = 1;
 
-	const VERSION = 1484;
+	const VERSION = 1507;
 
 	/**
 	 * Calls the post update functions
@@ -117,6 +118,12 @@ class PostUpdate
 		if (!self::update1484()) {
 			return false;
 		}
+		if (!self::update1506()) {
+			return false;
+		}
+		if (!self::update1507()) {
+			return false;
+		}
 		return true;
 	}
 
@@ -1184,4 +1191,116 @@ class PostUpdate
 
 		return false;
 	}
+
+	/**
+	 * update the "gsid" (global server id) field in the contact table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	private static function update1506()
+	{
+		// Was the script completed?
+		if (DI::keyValue()->get('post_update_version') >= 1506) {
+			return true;
+		}
+
+		$id = DI::keyValue()->get('post_update_version_1506_id') ?? 0;
+
+		Logger::info('Start', ['contact' => $id]);
+
+		$start_id = $id;
+		$rows = 0;
+		$condition = ["`id` > ? AND `gsid` IS NULL AND `network` = ?", $id, Protocol::DIASPORA];
+		$params = ['order' => ['id'], 'limit' => 10000];
+		$contacts = DBA::select('contact', ['id', 'url'], $condition, $params);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($contact = DBA::fetch($contacts)) {
+			$id = $contact['id'];
+
+			$parts = parse_url($contact['url']);
+			unset($parts['path']);
+			$server = (string)Uri::fromParts($parts);
+		
+			DBA::update('contact',
+				['gsid' => GServer::getID($server, true), 'baseurl' => GServer::cleanURL($server)],
+				['id' => $contact['id']]);
+
+			++$rows;
+		}
+		DBA::close($contacts);
+
+		DI::keyValue()->set('post_update_version_1506_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		if ($start_id == $id) {
+			DI::keyValue()->set('post_update_version', 1506);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * update the "gsid" (global server id) field in the inbox-status table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	private static function update1507()
+	{
+		// Was the script completed?
+		if (DI::keyValue()->get('post_update_version') >= 1507) {
+			return true;
+		}
+
+		$id = DI::keyValue()->get('post_update_version_1507_id') ?? '';
+
+		Logger::info('Start', ['apcontact' => $id]);
+
+		$start_id = $id;
+		$rows = 0;
+		$condition = ["`url` > ? AND NOT `gsid` IS NULL", $id];
+		$params = ['order' => ['url'], 'limit' => 10000];
+		$apcontacts = DBA::select('apcontact', ['url', 'gsid', 'sharedinbox', 'inbox'], $condition, $params);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($apcontact = DBA::fetch($apcontacts)) {
+			$id = $apcontact['url'];
+
+			$inbox = [$apcontact['inbox']];
+			if (!empty($apcontact['sharedinbox'])) {
+				$inbox[] = $apcontact['sharedinbox'];
+			}
+			$condition = DBA::mergeConditions(['url' => $inbox], ["`gsid` IS NULL"]);
+			DBA::update('inbox-status', ['gsid' => $apcontact['gsid'], 'archive' => GServer::isDefunctById($apcontact['gsid'])], $condition);
+			++$rows;
+		}
+		DBA::close($apcontacts);
+
+		DI::keyValue()->set('post_update_version_1507_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		if ($start_id == $id) {
+			DI::keyValue()->set('post_update_version', 1507);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
 }

src/Model/APContact.php

@@ -226,14 +226,11 @@ class APContact
 		$apcontact['following'] = JsonLD::fetchElement($compacted, 'as:following', '@id');
 		$apcontact['followers'] = JsonLD::fetchElement($compacted, 'as:followers', '@id');
 		$apcontact['inbox'] = (JsonLD::fetchElement($compacted, 'ldp:inbox', '@id') ?? '');
-		self::unarchiveInbox($apcontact['inbox'], false);
-
 		$apcontact['outbox'] = JsonLD::fetchElement($compacted, 'as:outbox', '@id');
 
 		$apcontact['sharedinbox'] = '';
 		if (!empty($compacted['as:endpoints'])) {
 			$apcontact['sharedinbox'] = (JsonLD::fetchElement($compacted['as:endpoints'], 'as:sharedInbox', '@id') ?? '');
-			self::unarchiveInbox($apcontact['sharedinbox'], true);
 		}
 
 		$apcontact['featured']      = JsonLD::fetchElement($compacted, 'toot:featured', '@id');
@@ -427,6 +424,12 @@ class APContact
 			$apcontact['gsid'] = null;
 		}
 
+		self::unarchiveInbox($apcontact['inbox'], false, $apcontact['gsid']);
+
+		if (!empty($apcontact['sharedinbox'])) {
+			self::unarchiveInbox($apcontact['sharedinbox'], true, $apcontact['gsid']);
+		}
+
 		if ($apcontact['url'] == $apcontact['alias']) {
 			$apcontact['alias'] = null;
 		}
@@ -517,7 +520,7 @@ class APContact
 	{
 		if (!empty($apcontact['inbox'])) {
 			Logger::info('Set inbox status to failure', ['inbox' => $apcontact['inbox']]);
-			HTTPSignature::setInboxStatus($apcontact['inbox'], false);
+			HTTPSignature::setInboxStatus($apcontact['inbox'], false, false, $apcontact['gsid']);
 		}
 
 		if (!empty($apcontact['sharedinbox'])) {
@@ -527,7 +530,7 @@ class APContact
 			if (!$available) {
 				// If all known personal inboxes are failing then set their shared inbox to failure as well
 				Logger::info('Set shared inbox status to failure', ['sharedinbox' => $apcontact['sharedinbox']]);
-				HTTPSignature::setInboxStatus($apcontact['sharedinbox'], false, true);
+				HTTPSignature::setInboxStatus($apcontact['sharedinbox'], false, true, $apcontact['gsid']);
 			}
 		}
 	}
@@ -542,11 +545,11 @@ class APContact
 	{
 		if (!empty($apcontact['inbox'])) {
 			Logger::info('Set inbox status to success', ['inbox' => $apcontact['inbox']]);
-			HTTPSignature::setInboxStatus($apcontact['inbox'], true);
+			HTTPSignature::setInboxStatus($apcontact['inbox'], true, false, $apcontact['gsid']);
 		}
 		if (!empty($apcontact['sharedinbox'])) {
 			Logger::info('Set shared inbox status to success', ['sharedinbox' => $apcontact['sharedinbox']]);
-			HTTPSignature::setInboxStatus($apcontact['sharedinbox'], true, true);
+			HTTPSignature::setInboxStatus($apcontact['sharedinbox'], true, true, $apcontact['gsid']);
 		}
 	}
 
@@ -555,15 +558,16 @@ class APContact
 	 *
 	 * @param string  $url    inbox url
 	 * @param boolean $shared Shared Inbox
+	 * @param int     $gsid   Global server id
 	 * @return void
 	 */
-	private static function unarchiveInbox(string $url, bool $shared)
+	private static function unarchiveInbox(string $url, bool $shared, int $gsid = null)
 	{
 		if (empty($url)) {
 			return;
 		}
 
-		HTTPSignature::setInboxStatus($url, true, $shared);
+		HTTPSignature::setInboxStatus($url, true, $shared, $gsid);
 	}
 
 	/**

src/Model/GServer.php

@@ -408,7 +408,7 @@ class GServer
 			['nurl' => Strings::normaliseLink($url)]);
 			Logger::info('Set failed status for existing server', ['url' => $url]);
 			if (self::isDefunct($gserver)) {
-				Contact::update(['archive' => true], ['gsid' => $gserver['id']]);
+				self::archiveContacts($gserver['id']);
 			}
 			return;
 		}
@@ -418,6 +418,18 @@ class GServer
 		Logger::info('Set failed status for new server', ['url' => $url]);
 	}
 
+	/**
+	 * Archive server related contacts and inboxes
+	 *
+	 * @param integer $gsid
+	 * @return void
+	 */
+	private static function archiveContacts(int $gsid)
+	{
+		Contact::update(['archive' => true], ['gsid' => $gsid]);
+		DBA::update('inbox-status', ['archive' => true], ['gsid' => $gsid]);
+	}
+
 	/**
 	 * Remove unwanted content from the given URL
 	 *

src/Util/HTTPSignature.php

@@ -332,15 +332,19 @@ class HTTPSignature
 	 * @param string  $url     The URL of the inbox
 	 * @param boolean $success Transmission status
 	 * @param boolean $shared  The inbox is a shared inbox
+	 * @param int     $gsid    Server ID
 	 * @throws \Exception
 	 */
-	static public function setInboxStatus(string $url, bool $success, bool $shared = false)
+	static public function setInboxStatus(string $url, bool $success, bool $shared = false, int $gsid = null)
 	{
 		$now = DateTimeFormat::utcNow();
 
 		$status = DBA::selectFirst('inbox-status', [], ['url' => $url]);
 		if (!DBA::isResult($status)) {
 			$insertFields = ['url' => $url, 'uri-id' => ItemURI::getIdByURI($url), 'created' => $now, 'shared' => $shared];
+			if (!empty($gsid)) {
+				$insertFields['gsid'] = $gsid;
+			}
 			if (!DBA::insert('inbox-status', $insertFields, Database::INSERT_IGNORE)) {
 				Logger::warning('Unable to insert inbox-status row', $insertFields);
 				return;
@@ -355,6 +359,10 @@ class HTTPSignature
 			$fields = ['failure' => $now];
 		}
 
+		if (!empty($gsid)) {
+			$fields['gsid'] = $gsid;
+		}
+
 		if ($status['failure'] > DBA::NULL_DATETIME) {
 			$new_previous_stamp = strtotime($status['failure']);
 			$old_previous_stamp = strtotime($status['previous']);

src/Worker/Notifier.php

@@ -816,7 +816,11 @@ class Notifier
 		}
 
 		// Fill the item cache
-		ActivityPub\Transmitter::createCachedActivityFromItem($target_item['id'], true);
+		$cache = ActivityPub\Transmitter::createCachedActivityFromItem($target_item['id'], true);
+		if (empty($cache)) {
+			Logger::info('Item cache was not created. The post will not be distributed.', ['id' => $target_item['id'], 'url' => $target_item['uri'], 'verb' => $target_item['verb']]);
+			return ['count' => 0, 'contacts' => []];
+		}
 
 		$delivery_queue_count = 0;
 		$contacts = [];

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1506);
+	define('DB_UPDATE_VERSION', 1507);
 }
 
 return [
@@ -872,6 +872,7 @@ return [
 		"fields" => [
 			"url" => ["type" => "varbinary(383)", "not null" => "1", "primary" => "1", "comment" => "URL of the inbox"],
 			"uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"],
+			"gsid" => ["type" => "int unsigned", "foreign" => ["gserver" => "id", "on delete" => "restrict"], "comment" => "ID of the related server"],
 			"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation date of this entry"],
 			"success" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last successful delivery"],
 			"failure" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last failed delivery"],
@@ -882,6 +883,7 @@ return [
 		"indexes" => [
 			"PRIMARY" => ["url"],
 			"uri-id" => ["uri-id"],
+			"gsid" => ["gsid"],
 		]
 	],
 	"intro" => [