From e548d647aaad807894c7e5a85014f94adeda356d Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 29 Oct 2020 05:20:26 +0000 Subject: [PATCH] New table for attached media files --- database.sql | 26 +++- src/Model/Item.php | 7 +- src/Model/Post/Media.php | 172 +++++++++++++++++++++++++ src/Protocol/ActivityPub/Processor.php | 74 +++++++++-- src/Protocol/ActivityPub/Receiver.php | 20 ++- src/Protocol/Diaspora.php | 32 ++++- static/dbstructure.config.php | 24 +++- 7 files changed, 329 insertions(+), 26 deletions(-) create mode 100644 src/Model/Post/Media.php diff --git a/database.sql b/database.sql index 1e439213d4..d3e75e753e 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2020.12-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1370 +-- DB_UPDATE_VERSION 1372 -- ------------------------------------------ @@ -775,6 +775,7 @@ CREATE TABLE IF NOT EXISTS `item-content` ( `title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title', `content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '', `body` mediumtext COMMENT 'item body content', + `raw-body` mediumtext COMMENT 'Body without embedded media links', `location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated', `coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated', `language` text COMMENT 'Language information about this post', @@ -1064,6 +1065,27 @@ CREATE TABLE IF NOT EXISTS `post-delivery-data` ( FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items'; +-- +-- TABLE post-media +-- +CREATE TABLE IF NOT EXISTS `post-media` ( + `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', + `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `url` varbinary(511) NOT NULL COMMENT 'Media URL', + `type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Media type', + `mimetype` varchar(60) COMMENT '', + `height` smallint unsigned COMMENT 'Height of the media', + `width` smallint unsigned COMMENT 'Width of the media', + `size` int unsigned COMMENT 'Media size', + `preview` varbinary(255) COMMENT 'Preview URL', + `preview-height` smallint unsigned COMMENT 'Height of the preview picture', + `preview-width` smallint unsigned COMMENT 'Width of the preview picture', + `description` text COMMENT '', + PRIMARY KEY(`id`), + UNIQUE INDEX `uri-id-url` (`uri-id`,`url`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Attached media'; + -- -- TABLE post-tag -- @@ -1390,7 +1412,7 @@ CREATE TABLE IF NOT EXISTS `workerqueue` ( PRIMARY KEY(`id`), INDEX `done_parameter` (`done`,`parameter`(64)), INDEX `done_executed` (`done`,`executed`), - INDEX `done_priority_created` (`done`,`priority`,`created`), + INDEX `done_priority_retrial_created` (`done`,`priority`,`retrial`,`created`), INDEX `done_priority_next_try` (`done`,`priority`,`next_try`), INDEX `done_pid_next_try` (`done`,`pid`,`next_try`), INDEX `done_pid_retrial` (`done`,`pid`,`retrial`), diff --git a/src/Model/Item.php b/src/Model/Item.php index aea428cd72..514163e96f 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -104,7 +104,7 @@ class Item 'object-type', 'object', 'target-type', 'target', 'plink']; // Field list for "item-content" table that is not present in the "item" table - const CONTENT_FIELDLIST = ['language']; + const CONTENT_FIELDLIST = ['language', 'raw-body']; // All fields in the item table const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', @@ -1678,6 +1678,7 @@ class Item $item['deny_gid'] = trim($item['deny_gid'] ?? ''); $item['private'] = intval($item['private'] ?? self::PUBLIC); $item['body'] = trim($item['body'] ?? ''); + $item['raw-body'] = trim($item['raw-body'] ?? $item['body']); $item['attach'] = trim($item['attach'] ?? ''); $item['app'] = trim($item['app'] ?? ''); $item['origin'] = intval($item['origin'] ?? 0); @@ -1816,6 +1817,10 @@ class Item self::setOwnerforResharedItem($item); } + // Remove all media attachments from the body and store them in the post-media table + $item['raw-body'] = Post\Media::addAttachmentsFromBody($item['uri-id'], $item['raw-body']); + $item['raw-body'] = self::setHashtags($item['raw-body']); + // Check for hashtags in the body and repair or add hashtag links $item['body'] = self::setHashtags($item['body']); diff --git a/src/Model/Post/Media.php b/src/Model/Post/Media.php new file mode 100644 index 0000000000..de50f13598 --- /dev/null +++ b/src/Model/Post/Media.php @@ -0,0 +1,172 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Core\Logger; +use Friendica\Core\System; +use Friendica\Database\DBA; +use Friendica\Util\Images; + +/** + * Class Media + * + * This Model class handles media interactions. + * This tables stores medias (images, videos, audio files) related to posts. + */ +class Media +{ + const UNKNOWN = 0; + const IMAGE = 1; + const VIDEO = 2; + const AUDIO = 3; + const TORRENT = 16; + + /** + * Insert a post-media record + * + * @param array $media + * @return void + */ + public static function insert(array $media) + { + if (empty($media['url']) || empty($media['uri-id'])) { + return; + } + + if (DBA::exists('post-media', ['uri-id' => $media['uri-id'], 'url' => $media['url']])) { + Logger::info('Media already exists', ['uri-id' => $media['uri-id'], 'url' => $media['url'], 'callstack' => System::callstack()]); + return; + } + + $fields = ['type', 'mimetype', 'height', 'width', 'size', 'preview', 'preview-height', 'preview-width', 'description']; + foreach ($fields as $field) { + if (empty($media[$field])) { + unset($media[$field]); + } + } + + if ($media['type'] == self::IMAGE) { + $imagedata = Images::getInfoFromURLCached($media['url']); + if (!empty($imagedata)) { + $media['mimetype'] = $imagedata['mime']; + $media['size'] = $imagedata['size']; + $media['width'] = $imagedata[0]; + $media['height'] = $imagedata[1]; + } + if (!empty($media['preview'])) { + $imagedata = Images::getInfoFromURLCached($media['preview']); + if (!empty($imagedata)) { + $media['preview-width'] = $imagedata[0]; + $media['preview-height'] = $imagedata[1]; + } + } + } + + $result = DBA::insert('post-media', $media, true); + Logger::info('Stored media', ['result' => $result, 'media' => $media, 'callstack' => System::callstack()]); + } + + /** + * Tests for path patterns that are usef for picture links in Friendica + * + * @param string $page Link to the image page + * @param string $preview Preview picture + * @return boolean + */ + private static function isPictureLink(string $page, string $preview) + { + return preg_match('#/photos/.*/image/#ism', $page) && preg_match('#/photo/.*-1\.#ism', $preview); + } + + /** + * Add media links and remove them from the body + * + * @param integer $uriid + * @param string $body + * @return string Body without media links + */ + public static function addAttachmentsFromBody(int $uriid, string $body) + { + // Simplify image codes + $body = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '[img]$3[/img]', $body); + + $attachments = []; + if (preg_match_all("#\[url=([^\]]+?)\]\s*\[img=([^\[\]]*)\]([^\[\]]*)\[\/img\]\s*\[/url\]#ism", $body, $pictures, PREG_SET_ORDER)) { + foreach ($pictures as $picture) { + if (!self::isPictureLink($picture[1], $picture[2])) { + continue; + } + $body = str_replace($picture[0], '', $body); + $image = str_replace('-1.', '-0.', $picture[2]); + $attachments[] = ['uri-id' => $uriid, 'type' => self::IMAGE, 'url' => $image, + 'preview' => $picture[2], 'description' => $picture[3]]; + } + } + + if (preg_match_all("/\[img=([^\[\]]*)\]([^\[\]]*)\[\/img\]/Usi", $body, $pictures, PREG_SET_ORDER)) { + foreach ($pictures as $picture) { + $body = str_replace($picture[0], '', $body); + $attachments[] = ['uri-id' => $uriid, 'type' => self::IMAGE, 'url' => $picture[1], 'description' => $picture[2]]; + } + } + + if (preg_match_all("#\[url=([^\]]+?)\]\s*\[img\]([^\[]+?)\[/img\]\s*\[/url\]#ism", $body, $pictures, PREG_SET_ORDER)) { + foreach ($pictures as $picture) { + if (!self::isPictureLink($picture[1], $picture[2])) { + continue; + } + $body = str_replace($picture[0], '', $body); + $image = str_replace('-1.', '-0.', $picture[2]); + $attachments[] = ['uri-id' => $uriid, 'type' => self::IMAGE, 'url' => $image, + 'preview' => $picture[2], 'description' => null]; + } + } + + if (preg_match_all("/\[img\]([^\[\]]*)\[\/img\]/ism", $body, $pictures, PREG_SET_ORDER)) { + foreach ($pictures as $picture) { + $body = str_replace($picture[0], '', $body); + $attachments[] = ['uri-id' => $uriid, 'type' => self::IMAGE, 'url' => $picture[1]]; + } + } + + /// @todo audio + video + if (preg_match_all("/\[audio\]([^\[\]]*)\[\/audio\]/ism", $body, $audios, PREG_SET_ORDER)) { + foreach ($audios as $audio) { + $body = str_replace($audio[0], '', $body); + $attachments[] = ['uri-id' => $uriid, 'type' => self::AUDIO, 'url' => $audio[1]]; + } + } + + if (preg_match_all("/\[video\]([^\[\]]*)\[\/video\]/ism", $body, $videos, PREG_SET_ORDER)) { + foreach ($videos as $video) { + $body = str_replace($video[0], '', $body); + $attachments[] = ['uri-id' => $uriid, 'type' => self::VIDEO, 'url' => $video[1]]; + } + } + + foreach ($attachments as $attachment) { + self::insert($attachment); + } + + return trim($body); + } +} diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index c7310d9eb8..7c8ec33d9d 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -37,6 +37,7 @@ use Friendica\Model\ItemURI; use Friendica\Model\Mail; use Friendica\Model\Tag; use Friendica\Model\User; +use Friendica\Model\Post; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Relay; @@ -81,6 +82,45 @@ class Processor return $body; } + /** + * Store attached media files in the post-media table + * + * @param int $uriid + * @param array $attachment + * @return void + */ + private static function storeAttachment(int $uriid, array $attachment) + { + if (empty($attachment['url'])) { + return; + } + + $data = ['uri-id' => $uriid]; + + $filetype = strtolower(substr($attachment['mediaType'], 0, strpos($attachment['mediaType'], '/'))); + if ($filetype == 'image') { + $data['type'] = Post\Media::IMAGE; + } elseif ($filetype == 'video') { + $data['type'] = Post\Media::VIDEO; + } elseif ($filetype == 'audio') { + $data['type'] = Post\Media::AUDIO; + } elseif (in_array($attachment['mediaType'], ['application/x-bittorrent', 'application/x-bittorrent;x-scheme-handler/magnet'])) { + $data['type'] = Post\Media::TORRENT; + } else { + Logger::info('Unknown type', ['attachment' => $attachment]); + return; + } + + $data['url'] = $attachment['url']; + $data['mimetype'] = $attachment['mediaType']; + $data['height'] = $attachment['height'] ?? null; + $data['size'] = $attachment['size'] ?? null; + $data['preview'] = $attachment['image'] ?? null; + $data['description'] = $attachment['name'] ?? null; + + Post\Media::insert($data); + } + /** * Add attachment data to the item array * @@ -95,6 +135,8 @@ class Processor return $item; } + $item['attach'] = ''; + foreach ($activity['attachments'] as $attach) { switch ($attach['type']) { case 'link': @@ -110,6 +152,8 @@ class Processor $item['body'] = PageInfo::appendDataToBody($item['body'], $data); break; default: + self::storeAttachment($item['uri-id'], $attach); + $filetype = strtolower(substr($attach['mediaType'], 0, strpos($attach['mediaType'], '/'))); if ($filetype == 'image') { if (!empty($activity['source']) && strpos($activity['source'], $attach['url'])) { @@ -146,13 +190,13 @@ class Processor $item['body'] .= "\n[video]" . $attach['url'] . '[/video]'; } else { - if (!empty($item["attach"])) { - $item["attach"] .= ','; + if (!empty($item['attach'])) { + $item['attach'] .= ','; } else { - $item["attach"] = ''; + $item['attach'] = ''; } - $item["attach"] .= '[attach]href="' . $attach['url'] . '" length="' . ($attach['length'] ?? '0') . '" type="' . $attach['mediaType'] . '" title="' . ($attach['name'] ?? '') . '"[/attach]'; + $item['attach'] .= '[attach]href="' . $attach['url'] . '" length="' . ($attach['length'] ?? '0') . '" type="' . $attach['mediaType'] . '" title="' . ($attach['name'] ?? '') . '"[/attach]'; } } } @@ -180,6 +224,9 @@ class Processor $item['edited'] = DateTimeFormat::utc($activity['updated']); $item = self::processContent($activity, $item); + + $item = self::constructAttachList($activity, $item); + if (empty($item)) { return; } @@ -403,17 +450,18 @@ class Processor { $item['title'] = HTML::toBBCode($activity['name']); + $content = HTML::toBBCode($activity['content']); + + if (!empty($activity['emojis'])) { + $content = self::replaceEmojis($content, $activity['emojis']); + } + + $content = self::convertMentions($content); + if (!empty($activity['source'])) { $item['body'] = $activity['source']; + $item['raw-body'] = $content; } else { - $content = HTML::toBBCode($activity['content']); - - if (!empty($activity['emojis'])) { - $content = self::replaceEmojis($content, $activity['emojis']); - } - - $content = self::convertMentions($content); - if (empty($activity['directmessage']) && ($item['thr-parent'] != $item['uri']) && ($item['gravity'] == GRAVITY_COMMENT)) { $item_private = !in_array(0, $activity['item_receiver']); $parent = Item::selectFirst(['id', 'uri-id', 'private', 'author-link', 'alias'], ['uri' => $item['thr-parent']]); @@ -429,7 +477,7 @@ class Processor $content = self::removeImplicitMentionsFromBody($content, $parent); } $item['content-warning'] = HTML::toBBCode($activity['summary']); - $item['body'] = $content; + $item['raw-body'] = $item['body'] = $content; } self::storeFromBody($item); diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 69d24a7abb..31a2dcb0b8 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -1231,24 +1231,36 @@ class Receiver $filetype = strtolower(substr($mediatype, 0, strpos($mediatype, '/'))); if ($filetype == 'audio') { - $attachments[$filetype] = ['type' => $mediatype, 'url' => $href]; + $attachments[$filetype] = ['type' => $mediatype, 'url' => $href, 'height' => null, 'size' => null]; } elseif ($filetype == 'video') { $height = (int)JsonLD::fetchElement($url, 'as:height', '@value'); + $size = (int)JsonLD::fetchElement($url, 'pt:size', '@value'); - // We save bandwidth by using a moderate height + // We save bandwidth by using a moderate height (alt least 480 pixel height) // Peertube normally uses these heights: 240, 360, 480, 720, 1080 if (!empty($attachments[$filetype]['height']) && - (($height > 480) || $height < $attachments[$filetype]['height'])) { + ($height > $attachments[$filetype]['height']) && ($attachments[$filetype]['height'] >= 480)) { continue; } - $attachments[$filetype] = ['type' => $mediatype, 'url' => $href, 'height' => $height]; + $attachments[$filetype] = ['type' => $mediatype, 'url' => $href, 'height' => $height, 'size' => $size]; + } elseif (in_array($mediatype, ['application/x-bittorrent', 'application/x-bittorrent;x-scheme-handler/magnet'])) { + $height = (int)JsonLD::fetchElement($url, 'as:height', '@value'); + + // For Torrent links we always store the highest resolution + if (!empty($attachments[$mediatype]['height']) && ($height < $attachments[$mediatype]['height'])) { + continue; + } + + $attachments[$mediatype] = ['type' => $mediatype, 'url' => $href, 'height' => $height, 'size' => null]; } } foreach ($attachments as $type => $attachment) { $object_data['attachments'][] = ['type' => $type, 'mediaType' => $attachment['type'], + 'height' => $attachment['height'], + 'size' => $attachment['size'], 'name' => '', 'url' => $attachment['url']]; } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index fd668b1f30..d2bbb7a4d0 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -2810,6 +2810,26 @@ class Diaspora return Relay::isSolicitedPost($tags, $body, $contact['id'], $url, Protocol::DIASPORA); } + /** + * Store an attached photo in the post-media table + * + * @param int $uriid + * @param object $photo + * @return void + */ + private static function storePhoto(int $uriid, $photo) + { + $data = []; + $data['uri-id'] = $uriid; + $data['type'] = Post\Media::IMAGE; + $data['url'] = XML::unescape($photo->remote_photo_path) . XML::unescape($photo->remote_photo_name); + $data['height'] = (int)XML::unescape($photo->height ?? 0); + $data['width'] = (int)XML::unescape($photo->width ?? 0); + $data['description'] = XML::unescape($photo->text ?? ''); + + Post\Media::insert($data); + } + /** * Receives status messages * @@ -2847,13 +2867,18 @@ class Diaspora } } - $body = Markdown::toBBCode($text); + $raw_body = $body = Markdown::toBBCode($text); $datarray = []; + $datarray["guid"] = $guid; + $datarray["uri"] = $datarray["parent-uri"] = self::getUriFromGuid($author, $guid); + $datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]); + // Attach embedded pictures to the body if ($data->photo) { foreach ($data->photo as $photo) { + self::storePhoto($datarray['uri-id'], $photo); $body = "[img]".XML::unescape($photo->remote_photo_path). XML::unescape($photo->remote_photo_name)."[/img]\n".$body; } @@ -2887,10 +2912,6 @@ class Diaspora $datarray["owner-link"] = $datarray["author-link"]; $datarray["owner-id"] = $datarray["author-id"]; - $datarray["guid"] = $guid; - $datarray["uri"] = $datarray["parent-uri"] = self::getUriFromGuid($author, $guid); - $datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]); - $datarray["verb"] = Activity::POST; $datarray["gravity"] = GRAVITY_PARENT; @@ -2904,6 +2925,7 @@ class Diaspora } $datarray["body"] = self::replacePeopleGuid($body, $contact["url"]); + $datarray["raw-body"] = self::replacePeopleGuid($raw_body, $contact["url"]); self::storeMentions($datarray['uri-id'], $text); Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]); diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 3322616785..a3b94e0b27 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -54,7 +54,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1371); + define('DB_UPDATE_VERSION', 1372); } return [ @@ -843,6 +843,7 @@ return [ "title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"], "content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "body" => ["type" => "mediumtext", "comment" => "item body content"], + "raw-body" => ["type" => "mediumtext", "comment" => "Body without embedded media links"], "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"], "coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"], "language" => ["type" => "text", "comment" => "Language information about this post"], @@ -1133,6 +1134,27 @@ return [ "PRIMARY" => ["uri-id"], ] ], + "post-media" => [ + "comment" => "Attached media", + "fields" => [ + "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], + "uri-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "url" => ["type" => "varbinary(511)", "not null" => "1", "comment" => "Media URL"], + "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Media type"], + "mimetype" => ["type" => "varchar(60)", "comment" => ""], + "height" => ["type" => "smallint unsigned", "comment" => "Height of the media"], + "width" => ["type" => "smallint unsigned", "comment" => "Width of the media"], + "size" => ["type" => "int unsigned", "comment" => "Media size"], + "preview" => ["type" => "varbinary(255)", "comment" => "Preview URL"], + "preview-height" => ["type" => "smallint unsigned", "comment" => "Height of the preview picture"], + "preview-width" => ["type" => "smallint unsigned", "comment" => "Width of the preview picture"], + "description" => ["type" => "text", "comment" => ""], + ], + "indexes" => [ + "PRIMARY" => ["id"], + "uri-id-url" => ["UNIQUE", "uri-id", "url"], + ] + ], "post-tag" => [ "comment" => "post relation to tags", "fields" => [