API: improved mail handling

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
--- Friendica 2021.06-dev (Siberian Iris)
+-- Friendica 2021.06-rc (Siberian Iris)
--- DB_UPDATE_VERSION 1418
+-- DB_UPDATE_VERSION 1419
 -- ------------------------------------------
 
 
@@ -751,6 +751,7 @@ CREATE TABLE IF NOT EXISTS `mail` (
 	`from-photo` varchar(255) NOT NULL DEFAULT '' COMMENT 'contact photo link of the sender',
 	`from-url` varchar(255) NOT NULL DEFAULT '' COMMENT 'profile linke of the sender',
 	`contact-id` varchar(255) COMMENT 'contact.id',
+	`author-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the author of the mail',
 	`convid` int unsigned COMMENT 'conv.id',
 	`title` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`body` mediumtext COMMENT '',
@@ -759,7 +760,11 @@ CREATE TABLE IF NOT EXISTS `mail` (
 	`replied` boolean NOT NULL DEFAULT '0' COMMENT '',
 	`unknown` boolean NOT NULL DEFAULT '0' COMMENT 'if sender not in the contact table this is 1',
 	`uri` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`uri-id` int unsigned COMMENT 'Item-uri id of the related mail',
 	`parent-uri` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`parent-uri-id` int unsigned COMMENT 'Item-uri id of the parent of the related mail',
+	`thr-parent` varchar(255) COMMENT '',
+	`thr-parent-id` int unsigned COMMENT 'Id of the item-uri table that contains the thread parent uri',
 	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'creation time of the private message',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid_seen` (`uid`,`seen`),
@@ -767,7 +772,15 @@ CREATE TABLE IF NOT EXISTS `mail` (
 	 INDEX `uri` (`uri`(64)),
 	 INDEX `parent-uri` (`parent-uri`(64)),
 	 INDEX `contactid` (`contact-id`(32)),
-	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
+	 INDEX `author-id` (`author-id`),
+	 INDEX `uri-id` (`uri-id`),
+	 INDEX `parent-uri-id` (`parent-uri-id`),
+	 INDEX `thr-parent-id` (`thr-parent-id`),
+	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT,
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`parent-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`thr-parent-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages';
 
 --

src/Content/Text/BBCode.php

@@ -1039,7 +1039,9 @@ class BBCode
 
 		switch ($simplehtml) {
 			case self::API:
-				$text = ($is_quote_share? '<br>' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' ' . $author_contact['addr'] . ': </p>' . "\n" . $content;
+				$text = ($is_quote_share? '<br>' : '') .
+				'<p><b><a href="' . $attributes['link'] . '">' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' ' . $author_contact['addr'] . "</a>:</b> </p>\n" .
+				'<blockquote class="shared_content">' . $content . '</blockquote>';
 				break;
 			case self::DIASPORA:
 				if (stripos(Strings::normaliseLink($attributes['link']), 'http://twitter.com/') === 0) {

src/Factory/Api/Mastodon/Status.php

@@ -136,7 +136,9 @@ class Status extends BaseFactory
 
 		$account = DI::mstdnAccount()->createFromContactId($item['author-id']);
 
-		$counts = new \Friendica\Object\Api\Mastodon\Status\Counts(0, 0, 0);
+		$replies = DBA::count('mail', ['thr-parent-id' => $item['uri-id'], 'reply' => true]);
+
+		$counts = new \Friendica\Object\Api\Mastodon\Status\Counts($replies, 0, 0);
 
 		$userAttributes = new \Friendica\Object\Api\Mastodon\Status\UserAttributes(false, false, false, false, false);
 

src/Model/Mail.php

@@ -71,6 +71,20 @@ class Mail
 			return false;
 		}
 
+		$msg['author-id']     = Contact::getIdForURL($msg['from-url'], 0, false);
+		$msg['uri-id']        = ItemURI::insert(['uri' => $msg['uri'], 'guid' => $msg['guid']]);
+		$msg['parent-uri-id'] = ItemURI::getIdByURI($msg['parent-uri']);
+
+		if ($msg['reply']) {
+			$reply = DBA::selectFirst('mail', ['uri', 'uri-id'], ['parent-uri' => $mail['parent-uri'], 'reply' => false]);
+
+			$msg['thr-parent']    = $reply['uri'];
+			$msg['thr-parent-id'] = $reply['uri-id'];
+		} else {
+			$msg['thr-parent']    = $msg['uri'];
+			$msg['thr-parent-id'] = $msg['uri-id'];
+		}
+
 		DBA::insert('mail', $msg);
 
 		$msg['id'] = DBA::lastInsertId();

src/Module/Api/Mastodon/Timelines/Direct.php

@@ -48,22 +48,22 @@ class Direct extends BaseApi
 			'limit'    => 20, // Maximum number of results to return. Defaults to 20.
 		]);
 
-		$params = ['order' => ['id' => true], 'limit' => $request['limit']];
+		$params = ['order' => ['uri-id' => true], 'limit' => $request['limit']];
 
 		$condition = ['uid' => $uid];
 
 		if (!empty($request['max_id'])) {
-			$condition = DBA::mergeConditions($condition, ["`id` < ?", $request['max_id']]);
+			$condition = DBA::mergeConditions($condition, ["`uri-id` < ?", $request['max_id']]);
 		}
 
 		if (!empty($request['since_id'])) {
-			$condition = DBA::mergeConditions($condition, ["`id` > ?", $request['since_id']]);
+			$condition = DBA::mergeConditions($condition, ["`uri-id` > ?", $request['since_id']]);
 		}
 
 		if (!empty($request['min_id'])) {
-			$condition = DBA::mergeConditions($condition, ["`id` > ?", $request['min_id']]);
+			$condition = DBA::mergeConditions($condition, ["`uri-id` > ?", $request['min_id']]);
 
-			$params['order'] = ['id'];
+			$params['order'] = ['uri-id'];
 		}
 
 		$mails = DBA::select('mail', ['id'], $condition, $params);

src/Protocol/ActivityPub/Transmitter.php

@@ -868,24 +868,19 @@ class Transmitter
 			return [];
 		}
 
-		$mail['uri-id'] = ItemURI::insert(['uri' => $mail['uri'], 'guid' => $mail['guid']]);
+		$reply = DBA::selectFirst('mail', ['uri', 'uri-id', 'from-url'], ['parent-uri' => $mail['parent-uri'], 'reply' => false]);
-
-		$reply = DBA::selectFirst('mail', ['uri', 'from-url', 'guid'], ['parent-uri' => $mail['parent-uri'], 'reply' => false]);
 
 		// Making the post more compatible for Mastodon by:
 		// - Making it a note and not an article (no title)
 		// - Moving the title into the "summary" field that is used as a "content warning"
 
-		if ($use_title) {
+		if (!$use_title) {
-			$mail['body']         = $mail['body'];
-			$mail['title']        = $mail['title'];
-		} else {
 			$mail['body']         = '[abstract]' . $mail['title'] . "[/abstract]\n" . $mail['body'];
 			$mail['title']        = '';
 		}
 
 		$mail['author-link']      = $mail['owner-link'] = $mail['from-url'];
-		$mail['author-id']        = Contact::getIdForURL($mail['author-link'], 0, false); 
+		$mail['owner-id']         = $mail['author-id'];
 		$mail['allow_cid']        = '<'.$mail['contact-id'].'>';
 		$mail['allow_gid']        = '';
 		$mail['deny_cid']         = '';
@@ -893,9 +888,9 @@ class Transmitter
 		$mail['private']          = Item::PRIVATE;
 		$mail['deleted']          = false;
 		$mail['edited']           = $mail['created'];
-		$mail['plink']            = $mail['uri'];
+		$mail['plink']            = DI::baseUrl() . '/message/' . $mail['id'];
-		$mail['thr-parent']       = $reply['uri'];
+		$mail['parent-uri']       = $reply['uri'];
-		$mail['thr-parent-id']    = ItemURI::insert(['uri' => $reply['uri'], 'guid' => $reply['guid']]);
+		$mail['parent-uri-id']    = $reply['uri-id'];
 		$mail['parent-author-id'] = Contact::getIdForURL($reply['from-url'], 0, false);
 		$mail['gravity']          = ($mail['reply'] ? GRAVITY_COMMENT: GRAVITY_PARENT);
 		$mail['event-type']       = '';

src/Worker/ExpirePosts.php

@@ -181,7 +181,10 @@ class ExpirePosts
 			AND NOT EXISTS(SELECT `uri-id` FROM `post-user` WHERE `uri-id` = `item-uri`.`id`)
 			AND NOT EXISTS(SELECT `parent-uri-id` FROM `post-user` WHERE `parent-uri-id` = `item-uri`.`id`)
 			AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`)
-			AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`)", $item['uri-id']]);
+			AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`)
+			AND NOT EXISTS(SELECT `uri-id` FROM `mail` WHERE `uri-id` = `item-uri`.`id`)
+			AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`)
+			AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]);
 
 		Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]);
 		$affected_count = 0;

src/Worker/RemoveContact.php

@@ -55,6 +55,7 @@ class RemoveContact {
 		}
 
 		DBA::delete('mail', ['contact-id' => $id]);
+		DBA::delete('mail', ['author-id' => $id]);
 
 		Post\ThreadUser::delete(['author-id' => $id]);
 		Post\ThreadUser::delete(['owner-id' => $id]);

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1418);
+	define('DB_UPDATE_VERSION', 1419);
 }
 
 return [
@@ -818,6 +818,7 @@ return [
 			"from-photo" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "contact photo link of the sender"],
 			"from-url" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "profile linke of the sender"],
 			"contact-id" => ["type" => "varchar(255)", "relation" => ["contact" => "id"], "comment" => "contact.id"],
+			"author-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the author of the mail"],
 			"convid" => ["type" => "int unsigned", "relation" => ["conv" => "id"], "comment" => "conv.id"],
 			"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
 			"body" => ["type" => "mediumtext", "comment" => ""],
@@ -826,7 +827,11 @@ return [
 			"replied" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
 			"unknown" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "if sender not in the contact table this is 1"],
 			"uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
+			"uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of the related mail"],
 			"parent-uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
+			"parent-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of the parent of the related mail"],
+			"thr-parent" => ["type" => "varchar(255)", "comment" => ""],
+			"thr-parent-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table that contains the thread parent uri"],
 			"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "creation time of the private message"],
 		],
 		"indexes" => [
@@ -836,6 +841,10 @@ return [
 			"uri" => ["uri(64)"],
 			"parent-uri" => ["parent-uri(64)"],
 			"contactid" => ["contact-id(32)"],
+			"author-id" => ["author-id"],
+			"uri-id" => ["uri-id"],
+			"parent-uri-id" => ["parent-uri-id"],
+			"thr-parent-id" => ["thr-parent-id"],
 		]
 	],
 	"mailacct" => [

update.php

@@ -49,6 +49,7 @@ use Friendica\Database\DBStructure;
 use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Item;
+use Friendica\Model\ItemURI;
 use Friendica\Model\Notification;
 use Friendica\Model\Photo;
 use Friendica\Model\Post;
@@ -912,3 +913,31 @@ function update_1413()
 		return Update::FAILED;
 	}
 }
+
+function update_1419()
+{
+	$mails = DBA::select('mail', ['id', 'from-url', 'uri', 'parent-uri', 'guid'], [], ['order' => ['id']]);
+	while ($mail = DBA::fetch($mails)) {
+		$fields = [];
+		$fields['author-id'] = Contact::getIdForURL($mail['from-url'], 0, false);
+		if (empty($fields['author-id'])) {
+			continue;
+		}
+
+		$fields['uri-id']        = ItemURI::insert(['uri' => $mail['uri'], 'guid' => $mail['guid']]);
+		$fields['parent-uri-id'] = ItemURI::getIdByURI($mail['parent-uri']);
+
+		$reply = DBA::selectFirst('mail', ['uri', 'uri-id', 'guid'], ['parent-uri' => $mail['parent-uri'], 'reply' => false]);
+		if (!empty($reply)) {
+			$fields['thr-parent'] = $reply['uri'];
+			if (!empty($reply['uri-id'])) {
+				$fields['thr-parent-id'] = $reply['uri-id'];
+			} else {
+				$fields['thr-parent-id'] = ItemURI::insert(['uri' => $reply['uri'], 'guid' => $reply['guid']]);
+			}
+		}
+
+		DBA::update('mail', $fields, ['id' => $mail['id']]);
+	}
+	return Update::SUCCESS;
+}