From aed5e4cc960797a25f184d24d7c81265e05d4ee6 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 5 Jul 2021 18:45:49 +0000 Subject: [PATCH] New struture for post related links --- database.sql | 17 ++++- doc/database.md | 1 + doc/database/db_post-link.md | 31 +++++++++ src/Model/Item.php | 15 ++++- src/Model/Post/Link.php | 122 ++++++++++++++++++++++++++++++++++ src/Module/Photo.php | 15 +++-- static/dbstructure.config.php | 15 ++++- 7 files changed, 206 insertions(+), 10 deletions(-) create mode 100644 doc/database/db_post-link.md create mode 100644 src/Model/Post/Link.php diff --git a/database.sql b/database.sql index b6dcb8ab7f..2121fec979 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ --- Friendica 2021.06-rc (Siberian Iris) --- DB_UPDATE_VERSION 1424 +-- Friendica 2021.09-dev (Siberian Iris) +-- DB_UPDATE_VERSION 1425 -- ------------------------------------------ @@ -1103,6 +1103,19 @@ CREATE TABLE IF NOT EXISTS `post-delivery-data` ( FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items'; +-- +-- TABLE post-link +-- +CREATE TABLE IF NOT EXISTS `post-link` ( + `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', + `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `url` varbinary(511) NOT NULL COMMENT 'External URL', + `mimetype` varchar(60) COMMENT '', + PRIMARY KEY(`id`), + UNIQUE INDEX `uri-id-url` (`uri-id`,`url`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Post related external links'; + -- -- TABLE post-media -- diff --git a/doc/database.md b/doc/database.md index 60e3253fe3..51f4ae5c6e 100644 --- a/doc/database.md +++ b/doc/database.md @@ -53,6 +53,7 @@ Database Tables | [post-category](help/database/db_post-category) | post relation to categories | | [post-content](help/database/db_post-content) | Content for all posts | | [post-delivery-data](help/database/db_post-delivery-data) | Delivery data for items | +| [post-link](help/database/db_post-link) | Post related external links | | [post-media](help/database/db_post-media) | Attached media | | [post-tag](help/database/db_post-tag) | post relation to tags | | [post-thread](help/database/db_post-thread) | Thread related data | diff --git a/doc/database/db_post-link.md b/doc/database/db_post-link.md new file mode 100644 index 0000000000..a162453567 --- /dev/null +++ b/doc/database/db_post-link.md @@ -0,0 +1,31 @@ +Table post-link +=========== + +Post related external links + +Fields +------ + +| Field | Description | Type | Null | Key | Default | Extra | +| -------- | --------------------------------------------------------- | -------------- | ---- | --- | ------- | -------------- | +| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | | NULL | | +| url | External URL | varbinary(511) | NO | | NULL | | +| mimetype | | varchar(60) | YES | | NULL | | + +Indexes +------------ + +| Name | Fields | +| ---------- | ------------------- | +| PRIMARY | id | +| uri-id-url | UNIQUE, uri-id, url | + +Foreign Keys +------------ + +| Field | Target Table | Target Field | +|-------|--------------|--------------| +| uri-id | [item-uri](help/database/db_item-uri) | id | + +Return to [database documentation](help/database) diff --git a/src/Model/Item.php b/src/Model/Item.php index a08b934d85..88c973efda 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -2755,6 +2755,7 @@ class Item } $attachments = Post\Media::splitAttachments($item['uri-id'], $item['guid'] ?? '', $shared_links); $item['body'] = self::replaceVisualAttachments($attachments, $item['body'] ?? ''); + $item['body'] = Post\Link::insertFromBody($item['uri-id'], $item['body']); $item['body'] = preg_replace("/\s*\[attachment .*?\].*?\[\/attachment\]\s*/ism", "\n", $item['body']); self::putInCache($item); @@ -2780,13 +2781,13 @@ class Item if (!empty($shared_attachments)) { $s = self::addVisualAttachments($shared_attachments, $item, $s, true); - $s = self::addLinkAttachment($shared_attachments, $body, $s, true, []); + $s = self::addLinkAttachment($shared_uri_id, $shared_attachments, $body, $s, true, []); $s = self::addNonVisualAttachments($shared_attachments, $item, $s, true); $body = preg_replace("/\s*\[share .*?\].*?\[\/share\]\s*/ism", '', $body); } $s = self::addVisualAttachments($attachments, $item, $s, false); - $s = self::addLinkAttachment($attachments, $body, $s, false, $shared_links); + $s = self::addLinkAttachment($item['uri-id'], $attachments, $body, $s, false, $shared_links); $s = self::addNonVisualAttachments($attachments, $item, $s, false); // Map. @@ -2967,7 +2968,7 @@ class Item * @param array $ignore_links A list of URLs to ignore * @return string modified content */ - private static function addLinkAttachment(array $attachments, string $body, string $content, bool $shared, array $ignore_links) + private static function addLinkAttachment(int $uriid, array $attachments, string $body, string $content, bool $shared, array $ignore_links) { $stamp1 = microtime(true); // @ToDo Check only for audio and video @@ -3052,6 +3053,14 @@ class Item } } + if (!empty($data['image'])) { + $data['image'] = Post\Link::getByLink($uriid, $data['image']); + } + + if (!empty($data['preview'])) { + $data['preview'] = Post\Link::getByLink($uriid, $data['preview']); + } + // @todo Use a template $rendered = BBCode::convertAttachment('', BBCode::INTERNAL, false, $data); } elseif (!self::containsLink($content, $data['url'], Post\Media::HTML)) { diff --git a/src/Model/Post/Link.php b/src/Model/Post/Link.php new file mode 100644 index 0000000000..83df038dac --- /dev/null +++ b/src/Model/Post/Link.php @@ -0,0 +1,122 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Core\Logger; +use Friendica\Database\DBA; +use Friendica\DI; +use Friendica\Util\Proxy; + +/** + * Class Link + * + * This Model class handles post related external links + */ +class Link +{ + public static function getByLink(int $uri_id, string $url, $size = '') + { + if (empty($url) || Proxy::isLocalImage($url)) { + return $url; + } + + $link = DBA::selectFirst('post-link', ['id'], ['uri-id' => $uri_id, 'url' => $url]); + if (!empty($link['id'])) { + $id = $link['id']; + Logger::info('Found', ['id' => $id, 'url' => $url]); + } else { + $mime = self::fetchMimeType($url); + + DBA::insert('post-link', ['uri-id' => $uri_id, 'url' => $url, 'mimetype' => $mime]); + $id = DBA::lastInsertId(); + Logger::info('Inserted', ['id' => $id, 'url' => $url]); + } + + if (empty($id)) { + return $url; + } + + $url = DI::baseUrl() . '/photo/link/'; + switch ($size) { + case Proxy::SIZE_MICRO: + $url .= Proxy::PIXEL_MICRO . '/'; + break; + case Proxy::SIZE_THUMB: + $url .= Proxy::PIXEL_THUMB . '/'; + break; + case Proxy::SIZE_SMALL: + $url .= Proxy::PIXEL_SMALL . '/'; + break; + case Proxy::SIZE_MEDIUM: + $url .= Proxy::PIXEL_MEDIUM . '/'; + break; + case Proxy::SIZE_LARGE: + $url .= Proxy::PIXEL_LARGE . '/'; + break; + } + return $url . $id; + } + + private static function fetchMimeType(string $url) + { + $timeout = DI::config()->get('system', 'xrd_timeout'); + $curlResult = DI::httpRequest()->head($url, ['timeout' => $timeout]); + if ($curlResult->isSuccess()) { + if (empty($media['mimetype'])) { + return $curlResult->getHeader('Content-Type'); + } + } + return ''; + } + + /** + * Add external links and replace them in the body + * + * @param integer $uriid + * @param string $body + * @return string Body with replaced links + */ + public static function insertFromBody(int $uriid, string $body) + { + if (preg_match_all("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", $body, $pictures, PREG_SET_ORDER)) { + foreach ($pictures as $picture) { + $body = str_replace($picture[3], self::getByLink($uriid, $picture[3]), $body); + } + } + + if (preg_match_all("/\[img=([^\[\]]*)\]([^\[\]]*)\[\/img\]/Usi", $body, $pictures, PREG_SET_ORDER)) { + foreach ($pictures as $picture) { + if (parse_url($picture[1], PHP_URL_SCHEME)) { + $body = str_replace($picture[1], self::getByLink($uriid, $picture[1]), $body); + } + } + } + + if (preg_match_all("/\[img\]([^\[\]]*)\[\/img\]/ism", $body, $pictures, PREG_SET_ORDER)) { + foreach ($pictures as $picture) { + $body = str_replace($picture[1], self::getByLink($uriid, $picture[1]), $body); + } + } + + return trim($body); + } +} diff --git a/src/Module/Photo.php b/src/Module/Photo.php index 44be9a31a2..e669de8a3d 100644 --- a/src/Module/Photo.php +++ b/src/Module/Photo.php @@ -176,7 +176,7 @@ class Photo extends BaseModule { switch($type) { case "preview": - $media = DBA::selectFirst('post-media', ['preview', 'url', 'type', 'uri-id'], ['id' => $uid]); + $media = DBA::selectFirst('post-media', ['preview', 'url', 'mimetype', 'type', 'uri-id'], ['id' => $uid]); if (empty($media)) { return false; } @@ -194,9 +194,9 @@ class Photo extends BaseModule return MPhoto::getPhoto($matches[1], $matches[2]); } - return MPhoto::createPhotoForExternalResource($url, (int)local_user()); + return MPhoto::createPhotoForExternalResource($url, (int)local_user(), $media['mimetype']); case "media": - $media = DBA::selectFirst('post-media', ['url', 'uri-id'], ['id' => $uid, 'type' => Post\Media::IMAGE]); + $media = DBA::selectFirst('post-media', ['url', 'mimetype', 'uri-id'], ['id' => $uid, 'type' => Post\Media::IMAGE]); if (empty($media)) { return false; } @@ -205,7 +205,14 @@ class Photo extends BaseModule return MPhoto::getPhoto($matches[1], $matches[2]); } - return MPhoto::createPhotoForExternalResource($media['url'], (int)local_user()); + return MPhoto::createPhotoForExternalResource($media['url'], (int)local_user(), $media['mimetype']); + case "link": + $link = DBA::selectFirst('post-link', ['url', 'mimetype'], ['id' => $uid]); + if (empty($link)) { + return false; + } + + return MPhoto::createPhotoForExternalResource($link['url'], (int)local_user(), $link['mimetype']); case "contact": $contact = Contact::getById($uid, ['uid', 'url', 'avatar', 'photo', 'xmpp', 'addr']); if (empty($contact)) { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index be939b1a17..c774b85705 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1424); + define('DB_UPDATE_VERSION', 1425); } return [ @@ -1151,6 +1151,19 @@ return [ "PRIMARY" => ["uri-id"], ] ], + "post-link" => [ + "comment" => "Post related external links", + "fields" => [ + "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], + "uri-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "url" => ["type" => "varbinary(511)", "not null" => "1", "comment" => "External URL"], + "mimetype" => ["type" => "varchar(60)", "comment" => ""], + ], + "indexes" => [ + "PRIMARY" => ["id"], + "uri-id-url" => ["UNIQUE", "uri-id", "url"], + ] + ], "post-media" => [ "comment" => "Attached media", "fields" => [