Merge pull request #12385 from annando/more-blurhash

Add more BlurHash to avoid not being able to display some picture

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2022.12-rc (Giant Rhubarb)
--- DB_UPDATE_VERSION 1500
+-- DB_UPDATE_VERSION 1501
 -- ------------------------------------------
 
 
@@ -129,6 +129,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
 	`xmpp` varchar(255) NOT NULL DEFAULT '' COMMENT 'XMPP address',
 	`matrix` varchar(255) NOT NULL DEFAULT '' COMMENT 'Matrix address',
 	`avatar` varbinary(383) NOT NULL DEFAULT '' COMMENT '',
+	`blurhash` varbinary(255) COMMENT 'BlurHash representation of the avatar',
 	`header` varbinary(383) COMMENT 'Header picture',
 	`url` varbinary(383) NOT NULL DEFAULT '' COMMENT '',
 	`nurl` varbinary(383) NOT NULL DEFAULT '' COMMENT '',
@@ -1297,6 +1298,9 @@ CREATE TABLE IF NOT EXISTS `post-link` (
 	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
 	`url` varbinary(511) NOT NULL COMMENT 'External URL',
 	`mimetype` varchar(60) COMMENT '',
+	`height` smallint unsigned COMMENT 'Height of the media',
+	`width` smallint unsigned COMMENT 'Width of the media',
+	`blurhash` varbinary(255) COMMENT 'BlurHash representation of the link',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `uri-id-url` (`uri-id`,`url`),
 	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE

doc/database/db_contact.md

@@ -21,6 +21,7 @@ Fields
 | xmpp                      | XMPP address                                                                                                   | varchar(255)       | NO   |     |                     |                |
 | matrix                    | Matrix address                                                                                                 | varchar(255)       | NO   |     |                     |                |
 | avatar                    |                                                                                                                | varbinary(383)     | NO   |     |                     |                |
+| blurhash                  | BlurHash representation of the avatar                                                                          | varbinary(255)     | YES  |     | NULL                |                |
 | header                    | Header picture                                                                                                 | varbinary(383)     | YES  |     | NULL                |                |
 | url                       |                                                                                                                | varbinary(383)     | NO   |     |                     |                |
 | nurl                      |                                                                                                                | varbinary(383)     | NO   |     |                     |                |

doc/database/db_post-link.md

@@ -6,12 +6,15 @@ Post related external links
 Fields
 ------
 
-| Field    | Description                                               | Type           | Null | Key | Default | Extra          |
+| Field    | Description                                               | Type              | Null | Key | Default | Extra          |
-| -------- | --------------------------------------------------------- | -------------- | ---- | --- | ------- | -------------- |
+| -------- | --------------------------------------------------------- | ----------------- | ---- | --- | ------- | -------------- |
-| id       | sequential ID                                             | int unsigned   | NO   | PRI | NULL    | auto_increment |
+| id       | sequential ID                                             | int unsigned      | NO   | PRI | NULL    | auto_increment |
-| uri-id   | Id of the item-uri table entry that contains the item uri | int unsigned   | NO   |     | NULL    |                |
+| uri-id   | Id of the item-uri table entry that contains the item uri | int unsigned      | NO   |     | NULL    |                |
-| url      | External URL                                              | varbinary(511) | NO   |     | NULL    |                |
+| url      | External URL                                              | varbinary(511)    | NO   |     | NULL    |                |
-| mimetype |                                                           | varchar(60)    | YES  |     | NULL    |                |
+| mimetype |                                                           | varchar(60)       | YES  |     | NULL    |                |
+| height   | Height of the media                                       | smallint unsigned | YES  |     | NULL    |                |
+| width    | Width of the media                                        | smallint unsigned | YES  |     | NULL    |                |
+| blurhash | BlurHash representation of the link                       | varbinary(255)    | YES  |     | NULL    |                |
 
 Indexes
 ------------

src/Contact/Avatar.php

@@ -90,6 +90,8 @@ class Avatar
 		$filename  = self::getFilename($contact['url']);
 		$timestamp = time();
 
+		$fields['blurhash'] = $image->getBlurHash();
+
 		$fields['photo'] = self::storeAvatarCache($image, $filename, Proxy::PIXEL_SMALL, $timestamp);
 		$fields['thumb'] = self::storeAvatarCache($image, $filename, Proxy::PIXEL_THUMB, $timestamp);
 		$fields['micro'] = self::storeAvatarCache($image, $filename, Proxy::PIXEL_MICRO, $timestamp);

src/Core/Storage/Type/ExternalResource.php

@@ -59,7 +59,7 @@ class ExternalResource implements ICanReadFromStorage
 		} catch (Exception $exception) {
 			throw new ReferenceStorageException(sprintf('External resource failed to get %s', $reference), $exception->getCode(), $exception);
 		}
-		if ($fetchResult->isSuccess()) {
+		if (!empty($fetchResult) && $fetchResult->isSuccess()) {
 			Logger::debug('Got picture', ['Content-Type' => $fetchResult->getHeader('Content-Type'), 'uid' => $data->uid, 'url' => $data->url]);
 			return $fetchResult->getBody();
 		} else {

src/Model/Contact.php

@@ -34,11 +34,15 @@ use Friendica\Core\Worker;
 use Friendica\Database\Database;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Network\HTTPClient\Client\HttpClientAccept;
+use Friendica\Network\HTTPClient\Client\HttpClientOptions;
 use Friendica\Network\HTTPException;
 use Friendica\Network\Probe;
+use Friendica\Object\Image;
 use Friendica\Protocol\Activity;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Util\DateTimeFormat;
+use Friendica\Util\HTTPSignature;
 use Friendica\Util\Images;
 use Friendica\Util\Network;
 use Friendica\Util\Proxy;
@@ -2193,7 +2197,7 @@ class Contact
 	 */
 	public static function updateAvatar(int $cid, string $avatar, bool $force = false, bool $create_cache = false)
 	{
-		$contact = DBA::selectFirst('contact', ['uid', 'avatar', 'photo', 'thumb', 'micro', 'xmpp', 'addr', 'nurl', 'url', 'network', 'uri-id'],
+		$contact = DBA::selectFirst('contact', ['uid', 'avatar', 'photo', 'thumb', 'micro', 'blurhash', 'xmpp', 'addr', 'nurl', 'url', 'network', 'uri-id'],
 			['id' => $cid, 'self' => false]);
 		if (!DBA::isResult($contact)) {
 			return;
@@ -2203,8 +2207,19 @@ class Contact
 
 		// Only update the cached photo links of public contacts when they already are cached
 		if (($uid == 0) && !$force && empty($contact['thumb']) && empty($contact['micro']) && !$create_cache) {
-			if ($contact['avatar'] != $avatar) {
+			if (($contact['avatar'] != $avatar) || empty($contact['blurhash'])) {
-				self::update(['avatar' => $avatar], ['id' => $cid]);
+				$update_fields = ['avatar' => $avatar];
+				$fetchResult = HTTPSignature::fetchRaw($avatar, 0, [HttpClientOptions::ACCEPT_CONTENT => [HttpClientAccept::IMAGE]]);
+
+				$img_str = $fetchResult->getBody();
+				if (!empty($img_str)) {
+					$image = new Image($img_str, Images::getMimeTypeByData($img_str));
+					if ($image->isValid()) {
+						$update_fields['blurhash'] = $image->getBlurHash();
+					}
+				}
+
+				self::update($update_fields, ['id' => $cid]);
 				Logger::info('Only update the avatar', ['id' => $cid, 'avatar' => $avatar, 'contact' => $contact]);
 			}
 			return;
@@ -2275,7 +2290,7 @@ class Contact
 				if ($update) {
 					$photos = Photo::importProfilePhoto($avatar, $uid, $cid, true);
 					if ($photos) {
-						$fields = ['avatar' => $avatar, 'photo' => $photos[0], 'thumb' => $photos[1], 'micro' => $photos[2], 'avatar-date' => DateTimeFormat::utcNow()];
+						$fields = ['avatar' => $avatar, 'photo' => $photos[0], 'thumb' => $photos[1], 'micro' => $photos[2], 'blurhash' => $photos[3], 'avatar-date' => DateTimeFormat::utcNow()];
 						$update = !empty($fields);
 						Logger::debug('Created new cached avatars', ['id' => $cid, 'uid' => $uid, 'owner-uid' => $local_uid]);
 					} else {

src/Model/Photo.php

@@ -313,6 +313,28 @@ class Photo
 		return $fields;
 	}
 
+	/**
+	 * Construct a photo array for a given image data string
+	 *
+	 * @param string $image_data Image data
+	 * @param string $mimetype   Image mime type. Is guessed by file name when empty.
+	 *
+	 * @return array
+	 * @throws \Exception
+	 */
+	public static function createPhotoForImageData(string $image_data, string $mimetype = ''): array
+	{
+		$fields = self::getFields();
+		$values = array_fill(0, count($fields), '');
+
+		$photo                  = array_combine($fields, $values);
+		$photo['data']          = $image_data;
+		$photo['type']          = $mimetype ?: Images::getMimeTypeByData($image_data);
+		$photo['cacheable']     = false;
+
+		return $photo;
+	}
+
 	/**
 	 * Construct a photo array for a system resource image
 	 *
@@ -647,7 +669,11 @@ class Photo
 			$micro = Contact::getDefaultAvatar($contact, Proxy::SIZE_MICRO);
 		}
 
-		return [$image_url, $thumb, $micro];
+		$photo = DBA::selectFirst(
+			'photo', ['blurhash'], ['uid' => $uid, 'contact-id' => $cid, 'scale' => 4, 'photo-type' => self::CONTACT_AVATAR]
+		);
+
+		return [$image_url, $thumb, $micro, $photo['blurhash']];
 	}
 
 	/**

src/Model/Post/Link.php

@@ -28,7 +28,10 @@ use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Network\HTTPClient\Client\HttpClientAccept;
 use Friendica\Network\HTTPClient\Client\HttpClientOptions;
+use Friendica\Util\HTTPSignature;
+use Friendica\Util\Images;
 use Friendica\Util\Proxy;
+use Friendica\Object\Image;
 
 /**
  * Class Link
@@ -72,12 +75,14 @@ class Link
 		if (!empty($link['id'])) {
 			$id = $link['id'];
 			Logger::info('Found', ['id' => $id, 'uri-id' => $uriId, 'url' => $url]);
-		} else {
+		} else { 
-			$mime = self::fetchMimeType($url);
+			$fields = self::fetchMimeType($url);
+			$fields['uri-id'] = $uriId;
+			$fields['url'] = $url;
 
-			DBA::insert('post-link', ['uri-id' => $uriId, 'url' => $url, 'mimetype' => $mime], Database::INSERT_IGNORE);
+			DBA::insert('post-link', $fields, Database::INSERT_IGNORE);
 			$id = DBA::lastInsertId();
-			Logger::info('Inserted', ['id' => $id, 'uri-id' => $uriId, 'url' => $url]);
+			Logger::info('Inserted', $fields);
 		}
 
 		if (empty($id)) {
@@ -114,19 +119,28 @@ class Link
 	 *
 	 * @param string $url URL to fetch
 	 * @param string $accept Comma-separated list of expected response MIME type(s)
-	 * @return string Discovered MIME type or empty string on failure
+	 * @return array Discovered MIME type and blurhash or empty array on failure
 	 */
-	private static function fetchMimeType(string $url, string $accept = HttpClientAccept::DEFAULT): string
+	private static function fetchMimeType(string $url, string $accept = HttpClientAccept::DEFAULT): array
 	{
 		$timeout = DI::config()->get('system', 'xrd_timeout');
 
-		$curlResult = DI::httpClient()->head($url, [HttpClientOptions::TIMEOUT => $timeout, HttpClientOptions::ACCEPT_CONTENT => $accept]);
+		$curlResult = HTTPSignature::fetchRaw($url, 0, [HttpClientOptions::TIMEOUT => $timeout, HttpClientOptions::ACCEPT_CONTENT => $accept]);
+		if (!$curlResult->isSuccess()) {
+			return [];
+		}
+		$fields = ['mimetype' => $curlResult->getHeader('Content-Type')[0]];
 
-		if ($curlResult->isSuccess() && empty($media['mimetype'])) {
+		$img_str = $curlResult->getBody();
-			return $curlResult->getHeader('Content-Type')[0] ?? '';
+		$image = new Image($img_str, Images::getMimeTypeByData($img_str));
+		if ($image->isValid()) {
+			$fields['mimetype'] = $image->getType();
+			$fields['width']    = $image->getWidth();
+			$fields['height']   = $image->getHeight();
+			$fields['blurhash'] = $image->getBlurHash();
 		}
 
-		return '';
+		return $fields;
 	}
 
 	/**

src/Module/Photo.php

@@ -285,14 +285,14 @@ class Photo extends BaseModule
 
 				return MPhoto::createPhotoForExternalResource($media['url'], (int)DI::userSession()->getLocalUserId(), $media['mimetype'], $media['blurhash'], $media['width'], $media['height']);
 			case 'link':
-				$link = DBA::selectFirst('post-link', ['url', 'mimetype'], ['id' => $id]);
+				$link = DBA::selectFirst('post-link', ['url', 'mimetype', 'blurhash', 'width', 'height'], ['id' => $id]);
 				if (empty($link)) {
 					return false;
 				}
 
-				return MPhoto::createPhotoForExternalResource($link['url'], (int)DI::userSession()->getLocalUserId(), $link['mimetype'] ?? '');
+				return MPhoto::createPhotoForExternalResource($link['url'], (int)DI::userSession()->getLocalUserId(), $link['mimetype'] ?? '', $link['blurhash'] ?? '', $link['width'] ?? 0, $link['height'] ?? 0);
 			case 'contact':
-				$fields = ['uid', 'uri-id', 'url', 'nurl', 'avatar', 'photo', 'xmpp', 'addr', 'network', 'failed', 'updated'];
+				$fields = ['uid', 'uri-id', 'url', 'nurl', 'avatar', 'photo', 'blurhash', 'xmpp', 'addr', 'network', 'failed', 'updated'];
 				$contact = Contact::getById($id, $fields);
 				if (empty($contact)) {
 					return false;
@@ -364,7 +364,11 @@ class Photo extends BaseModule
 						Logger::debug('Expected Content-Type', ['mime' => $mimetext, 'url' => $url]);
 					}
 				}
-				if (empty($mimetext)) {
+				if (empty($mimetext) && !empty($contact['blurhash'])) {
+					$image = New Image('', 'image/png');
+					$image->getFromBlurHash($contact['blurhash'], $customsize, $customsize);
+					return MPhoto::createPhotoForImageData($image->asString());
+				} elseif (empty($mimetext)) {
 					if ($customsize <= Proxy::PIXEL_MICRO) {
 						$url = Contact::getDefaultAvatar($contact ?: [], Proxy::SIZE_MICRO);
 					} elseif ($customsize <= Proxy::PIXEL_THUMB) {
@@ -373,7 +377,7 @@ class Photo extends BaseModule
 						$url = Contact::getDefaultAvatar($contact ?: [], Proxy::SIZE_SMALL);
 					}
 				}
-				return MPhoto::createPhotoForExternalResource($url, 0, $mimetext);
+				return MPhoto::createPhotoForExternalResource($url, 0, $mimetext, $contact['blurhash'], $customsize, $customsize);
 			case 'header':
 				$fields = ['uid', 'url', 'header', 'network', 'gsid'];
 				$contact = Contact::getById($id, $fields);

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1500);
+	define('DB_UPDATE_VERSION', 1501);
 }
 
 return [
@@ -186,6 +186,7 @@ return [
 			"xmpp" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "XMPP address"],
 			"matrix" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Matrix address"],
 			"avatar" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""],
+			"blurhash" => ["type" => "varbinary(255)", "comment" => "BlurHash representation of the avatar"],
 			"header" => ["type" => "varbinary(383)", "comment" => "Header picture"],
 			"url" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""],
 			"nurl" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""],
@@ -1323,6 +1324,9 @@ return [
 			"uri-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
 			"url" => ["type" => "varbinary(511)", "not null" => "1", "comment" => "External URL"],
 			"mimetype" => ["type" => "varchar(60)", "comment" => ""],
+			"height" => ["type" => "smallint unsigned", "comment" => "Height of the media"],
+			"width" => ["type" => "smallint unsigned", "comment" => "Width of the media"],
+			"blurhash" => ["type" => "varbinary(255)", "comment" => "BlurHash representation of the link"],
 		],
 		"indexes" => [
 			"PRIMARY" => ["id"],