strk/friendica
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Support for page JSON-LD based page information

Browse source
This commit is contained in:
Michael 2021-03-15 22:02:21 +00:00
parent ddbe8730ed
commit 70bf75c342
3 changed files with 355 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
src/Content/OEmbed.php
View file

@ -143,35 +143,54 @@ class OEmbed
DI::cache()->set($cache_key, $json_string, $cache_ttl); DI::cache()->set($cache_key, $json_string, $cache_ttl);
} }
if ($oembed->type == 'error') { // Always embed the SSL version
if (!empty($oembed->html)) {
$oembed->html = str_replace(['http://www.youtube.com/', 'http://player.vimeo.com/'], ['https://www.youtube.com/', 'https://player.vimeo.com/'], $oembed->html);
}
// Improve the OEmbed data with data from OpenGraph, Twitter cards and other sources
$data = ParseUrl::getSiteinfoCached($embedurl, true, false);
if (($oembed->type == 'error') && empty($data['title']) && empty($data['text'])) {
return $oembed; return $oembed;
} }
// Always embed the SSL version if ($no_rich_type || ($oembed->type == 'error')) {
$oembed->html = str_replace(['http://www.youtube.com/', 'http://player.vimeo.com/'], ['https://www.youtube.com/', 'https://player.vimeo.com/'], $oembed->html); $oembed->html = '';
// If fetching information doesn't work, then improve via internal functions
if ($no_rich_type && ($oembed->type == 'rich')) {
$data = ParseUrl::getSiteinfoCached($embedurl, true, false);
$oembed->type = $data['type']; $oembed->type = $data['type'];
if ($oembed->type == 'photo') { if ($oembed->type == 'photo') {
$oembed->url = $data['url']; $oembed->url = $data['url'];
} }
}
if (isset($data['title'])) { if (!empty($data['title']) && empty($oembed->title)) {
$oembed->title = $data['title']; $oembed->title = $data['title'];
} }
if (isset($data['text'])) { if (!empty($data['text']) && empty($oembed->description)) {
$oembed->description = $data['text']; $oembed->description = $data['text'];
} }
if (!empty($data['images'])) { if (!empty($data['publisher']) && empty($oembed->provider_name)) {
$oembed->thumbnail_url = $data['images'][0]['src']; $oembed->provider_name = $data['publisher'];
$oembed->thumbnail_width = $data['images'][0]['width']; }
$oembed->thumbnail_height = $data['images'][0]['height'];
} if (!empty($data['publisher_url']) && empty($oembed->provider_url)) {
$oembed->provider_url = $data['publisher_url'];
}
if (!empty($data['author']) && empty($oembed->author_name)) {
$oembed->author_name = $data['author'];
}
if (!empty($data['author_url']) && empty($oembed->author_url)) {
$oembed->author_url = $data['author_url'];
}
if (!empty($data['images']) && empty($oembed->thumbnail_url)) {
$oembed->thumbnail_url = $data['images'][0]['src'];
$oembed->thumbnail_width = $data['images'][0]['width'];
$oembed->thumbnail_height = $data['images'][0]['height'];
} }
Hook::callAll('oembed_fetch_url', $embedurl, $oembed); Hook::callAll('oembed_fetch_url', $embedurl, $oembed);

17
src/Content/PageInfo.php
View file

@ -129,17 +129,12 @@ class PageInfo
} }
// Escape some bad characters // Escape some bad characters
$data['url'] = str_replace(['[', ']'], ['[', ']'], htmlentities($data['url'], ENT_QUOTES, 'UTF-8', false)); $text = "[attachment";
$data['title'] = str_replace(['[', ']'], ['[', ']'], htmlentities($data['title'], ENT_QUOTES, 'UTF-8', false));
$text = "[attachment type='" . $data['type'] . "'"; foreach (['type', 'url', 'title', 'alternative_title', 'publisher', 'publisher_url', 'publisher_image', 'author', 'author_url', 'author_image'] as $field) {
if (!empty($data[$field])) {
if (!empty($data['url'])) { $text .= " " . $field . "='" . str_replace(['[', ']'], ['[', ']'], htmlentities($data[$field], ENT_QUOTES, 'UTF-8', false)) . "'";
$text .= " url='" . $data['url'] . "'"; }
}
if (!empty($data['title'])) {
$text .= " title='" . $data['title'] . "'";
} }
if (empty($data['text'])) { if (empty($data['text'])) {
@ -167,7 +162,7 @@ class PageInfo
} }
} }
$text .= ']' . $data['text'] . '[/attachment]'; $text .= ']' . str_replace(['[', ']'], ['[', ']'], $data['text']) . '[/attachment]';
$hashtags = ''; $hashtags = '';
if (!empty($data['keywords'])) { if (!empty($data['keywords'])) {

313
src/Util/ParseUrl.php
View file

@ -300,7 +300,6 @@ class ParseUrl
@$doc->loadHTML($body); @$doc->loadHTML($body);
XML::deleteNode($doc, 'style'); XML::deleteNode($doc, 'style');
XML::deleteNode($doc, 'script');
XML::deleteNode($doc, 'option'); XML::deleteNode($doc, 'option');
XML::deleteNode($doc, 'h1'); XML::deleteNode($doc, 'h1');
XML::deleteNode($doc, 'h2'); XML::deleteNode($doc, 'h2');
@ -392,6 +391,9 @@ class ParseUrl
case 'dc.description': case 'dc.description':
$siteinfo['text'] = trim($meta_tag['content']); $siteinfo['text'] = trim($meta_tag['content']);
break; break;
case 'dc.creator':
$siteinfo['publisher'] = trim($meta_tag['content']);
break;
case 'keywords': case 'keywords':
$keywords = explode(',', $meta_tag['content']); $keywords = explode(',', $meta_tag['content']);
break; break;
@ -426,12 +428,46 @@ class ParseUrl
case 'og:image': case 'og:image':
$siteinfo['image'] = $meta_tag['content']; $siteinfo['image'] = $meta_tag['content'];
break; break;
case 'og:image:url':
$siteinfo['image'] = $meta_tag['content'];
break;
case 'og:image:secure_url':
$siteinfo['image'] = $meta_tag['content'];
break;
case 'og:title': case 'og:title':
$siteinfo['title'] = trim($meta_tag['content']); $siteinfo['title'] = trim($meta_tag['content']);
break; break;
case 'og:description': case 'og:description':
$siteinfo['text'] = trim($meta_tag['content']); $siteinfo['text'] = trim($meta_tag['content']);
break; break;
case 'og:site_name':
$siteinfo['publisher'] = trim($meta_tag['content']);
break;
case 'twitter:description':
$siteinfo['text'] = trim($meta_tag['content']);
break;
case 'twitter:title':
$siteinfo['title'] = trim($meta_tag['content']);
break;
case 'twitter:image':
$siteinfo['image'] = $meta_tag['content'];
break;
}
}
}
$list = $xpath->query("//script[@type='application/ld+json']");
foreach ($list as $node) {
if (!empty($node->nodeValue)) {
$nodevalue = html_entity_decode($node->nodeValue, ENT_COMPAT, 'UTF-8');
if ($jsonld = json_decode($nodevalue, true)) {
if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) {
foreach ($jsonld['@graph'] as $part) {
$siteinfo = self::parseJsonLd($siteinfo, $part);
}
} else {
$siteinfo = self::parseJsonLd($siteinfo, $jsonld);
}
} }
} }
} }
@ -470,6 +506,281 @@ class ParseUrl
return $siteinfo; return $siteinfo;
} }
/**
* Improve the siteinfo with information from the provided JSON-LD information
* @see https://jsonld.com/
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLd(array $siteinfo, array $jsonld)
{
$type = JsonLD::fetchElement($jsonld, '@type');
switch ($type) {
case 'Article':
case 'NewsArticle':
return self::parseJsonLdArticle($siteinfo, $jsonld);
case 'WebPage':
return self::parseJsonLdWebPage($siteinfo, $jsonld);
case 'WebSite':
return self::parseJsonLdWebSite($siteinfo, $jsonld);
case 'Organization':
return self::parseJsonLdWebOrganization($siteinfo, $jsonld);
case 'Person':
return self::parseJsonLdWebPerson($siteinfo, $jsonld);
case 'BreadcrumbList':
case 'Audio': /// @todo Can contain direct media links to audio - can be interesting in the future
case 'VideoObject':
case 'ImageObject':
case 'LiveBlogPosting':
case 'SocialMediaPosting':
// quit silently
return $siteinfo;
default:
Logger::info('Unsupported or unknown type', ['type' => $type, 'url' => $siteinfo['url']]);
return $siteinfo;
}
}
/**
* Improve the siteinfo with information from the provided JSON-LD information concerning authors and publishers
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdAuthor(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
if (is_array($jsonld['publisher'])) {
$content = JsonLD::fetchElement($jsonld, 'publisher', 'name', '@type', 'Organization');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'publisher', 'url', '@type', 'Organization');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$brand = JsonLD::fetchElement($jsonld, 'publisher', 'brand', '@type', 'Organization');
if (!empty($brand)) {
$content = JsonLD::fetchElement($brand, 'name', '@type', 'brand');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher'] = trim($content);
}
}
}
if (is_array($jsonld['author'])) {
$content = JsonLD::fetchElement($jsonld, 'author', 'name', '@type', 'Organization');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'author', 'url', '@type', 'Organization');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'author', 'name', '@type', 'Person');
if (!empty($content) && is_string($content)) {
$jsonldinfo['author'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'author', 'url', '@type', 'Person');
if (!empty($content) && is_string($content)) {
$jsonldinfo['author_url'] = trim($content);
}
}
Logger::info('Fetched author information', ['fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Improve the siteinfo with information from the provided JSON-LD Article information
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdArticle(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'headline');
if (!empty($content) && is_string($content)) {
$jsonldinfo['title'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'alternativeHeadline');
if (!empty($content) && is_string($content)) {
$jsonldinfo['alternative_title'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content) && is_string($content)) {
$jsonldinfo['text'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject');
if (!empty($content) && is_string($content)) {
$jsonldinfo['image'] = trim($content);
}
/// @todo Check for the correct separator, also check for dpublicates before adding
// $content = JsonLD::fetchElement($jsonld, 'keywords');
// if (!empty($content) && is_string($content)) {
// $jsonldinfo['keywords'] = trim($content);
// }
$jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
Logger::info('Fetched article information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Improve the siteinfo with information from the provided JSON-LD WebPage information
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdWebPage(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content)) {
$jsonldinfo['title'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content)) {
$jsonldinfo['text'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
if (!empty($content)) {
$jsonldinfo['image'] = trim($content);
}
$jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
Logger::info('Fetched webpage information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Improve the siteinfo with information from the provided JSON-LD WebSite information
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdWebSite(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content)) {
$jsonldinfo['publisher'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content)) {
$jsonldinfo['publisher_description'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
if (!empty($content)) {
$jsonldinfo['image'] = trim($content);
}
$jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
Logger::info('Fetched WebSite information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Improve the siteinfo with information from the provided JSON-LD Organization information
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content)) {
$jsonldinfo['publisher'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'logo', 'url', '@type', 'ImageObject');
if (!empty($content)) {
$jsonldinfo['publisher_image'] = trim($content);
}
Logger::info('Fetched Organization information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Improve the siteinfo with information from the provided JSON-LD Person information
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content)) {
$jsonldinfo['author'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content)) {
$jsonldinfo['author_description'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
$jsonldinfo['author_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject');
if (!empty($content)) {
$jsonldinfo['author_image'] = trim($content);
}
Logger::info('Fetched Person information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/** /**
* Convert tags from CSV to an array * Convert tags from CSV to an array
* *