strk/friendica
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #10054 from annando/json-ld-page

Browse source 
use Json-Ld data to fetch more data for a page
This commit is contained in:
Hypolite Petovan 2021-03-18 10:56:57 -04:00 committed by GitHub
parent 8e12bb2de5 f772e11cfa
commit d4e8d82252
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 632 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
mod/item.php
View file

@ -56,6 +56,7 @@ use Friendica\Protocol\Activity;
use Friendica\Protocol\Diaspora; use Friendica\Protocol\Diaspora;
use Friendica\Security\Security; use Friendica\Security\Security;
use Friendica\Util\DateTimeFormat; use Friendica\Util\DateTimeFormat;
use Friendica\Util\ParseUrl;
use Friendica\Worker\Delivery; use Friendica\Worker\Delivery;
function item_post(App $a) { function item_post(App $a) {
@ -217,12 +218,15 @@ function item_post(App $a) {
$attachment_img_width = $_REQUEST['attachment_img_width'] ?? 0; $attachment_img_width = $_REQUEST['attachment_img_width'] ?? 0;
$attachment_img_height = $_REQUEST['attachment_img_height'] ?? 0; $attachment_img_height = $_REQUEST['attachment_img_height'] ?? 0;
$attachment = [
'type' => $attachment_type, // Fetch the basic attachment data
'title' => $attachment_title, $attachment = ParseUrl::getSiteinfoCached($attachment_url);
'text' => $attachment_text,
'url' => $attachment_url, // Overwrite the basic data with possible changes from the frontend
]; $attachment['type'] = $attachment_type;
$attachment['title'] = $attachment_title;
$attachment['text'] = $attachment_text;
$attachment['url'] = $attachment_url;
if (!empty($attachment_img_src)) { if (!empty($attachment_img_src)) {
$attachment['images'] = [ $attachment['images'] = [
@ -232,6 +236,8 @@ function item_post(App $a) {
'height' => $attachment_img_height 'height' => $attachment_img_height
] ]
]; ];
} else {
unset($attachment['images']);
} }
$att_bbcode = "\n" . PageInfo::getFooterFromData($attachment); $att_bbcode = "\n" . PageInfo::getFooterFromData($attachment);

43
src/Content/OEmbed.php
View file

@ -62,11 +62,12 @@ class OEmbed
* *
* @param string $embedurl The URL from which the data should be fetched. * @param string $embedurl The URL from which the data should be fetched.
* @param bool $no_rich_type If set to true rich type content won't be fetched. * @param bool $no_rich_type If set to true rich type content won't be fetched.
* @param bool $use_parseurl Use the "ParseUrl" functionality to add additional data
* *
* @return \Friendica\Object\OEmbed * @return \Friendica\Object\OEmbed
* @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/ */
public static function fetchURL($embedurl, $no_rich_type = false) public static function fetchURL($embedurl, bool $no_rich_type = false, bool $use_parseurl = true)
{ {
$embedurl = trim($embedurl, '\'"'); $embedurl = trim($embedurl, '\'"');
@ -143,30 +144,52 @@ class OEmbed
DI::cache()->set($cache_key, $json_string, $cache_ttl); DI::cache()->set($cache_key, $json_string, $cache_ttl);
} }
if ($oembed->type == 'error') { // Always embed the SSL version
if (!empty($oembed->html)) {
$oembed->html = str_replace(['http://www.youtube.com/', 'http://player.vimeo.com/'], ['https://www.youtube.com/', 'https://player.vimeo.com/'], $oembed->html);
}
// Improve the OEmbed data with data from OpenGraph, Twitter cards and other sources
if ($use_parseurl) {
$data = ParseUrl::getSiteinfoCached($embedurl, false);
if (($oembed->type == 'error') && empty($data['title']) && empty($data['text'])) {
return $oembed; return $oembed;
} }
// Always embed the SSL version if ($no_rich_type || ($oembed->type == 'error')) {
$oembed->html = str_replace(['http://www.youtube.com/', 'http://player.vimeo.com/'], ['https://www.youtube.com/', 'https://player.vimeo.com/'], $oembed->html); $oembed->html = '';
// If fetching information doesn't work, then improve via internal functions
if ($no_rich_type && ($oembed->type == 'rich')) {
$data = ParseUrl::getSiteinfoCached($embedurl, true, false);
$oembed->type = $data['type']; $oembed->type = $data['type'];
if ($oembed->type == 'photo') { if ($oembed->type == 'photo') {
$oembed->url = $data['url']; $oembed->url = $data['url'];
} }
}
if (isset($data['title'])) { if (!empty($data['title'])) {
$oembed->title = $data['title']; $oembed->title = $data['title'];
} }
if (isset($data['text'])) { if (!empty($data['text'])) {
$oembed->description = $data['text']; $oembed->description = $data['text'];
} }
if (!empty($data['publisher_name'])) {
$oembed->provider_name = $data['publisher_name'];
}
if (!empty($data['publisher_url'])) {
$oembed->provider_url = $data['publisher_url'];
}
if (!empty($data['author_name'])) {
$oembed->author_name = $data['author_name'];
}
if (!empty($data['author_url'])) {
$oembed->author_url = $data['author_url'];
}
if (!empty($data['images'])) { if (!empty($data['images'])) {
$oembed->thumbnail_url = $data['images'][0]['src']; $oembed->thumbnail_url = $data['images'][0]['src'];
$oembed->thumbnail_width = $data['images'][0]['width']; $oembed->thumbnail_width = $data['images'][0]['width'];

17
src/Content/PageInfo.php
View file

@ -129,17 +129,12 @@ class PageInfo
} }
// Escape some bad characters // Escape some bad characters
$data['url'] = str_replace(['[', ']'], ['[', ']'], htmlentities($data['url'], ENT_QUOTES, 'UTF-8', false)); $text = "[attachment";
$data['title'] = str_replace(['[', ']'], ['[', ']'], htmlentities($data['title'], ENT_QUOTES, 'UTF-8', false));
$text = "[attachment type='" . $data['type'] . "'"; foreach (['type', 'url', 'title', 'alternative_title', 'publisher_name', 'publisher_url', 'publisher_img', 'author_name', 'author_url', 'author_img'] as $field) {
if (!empty($data[$field])) {
if (!empty($data['url'])) { $text .= " " . $field . "='" . str_replace(['[', ']'], ['[', ']'], htmlentities($data[$field], ENT_QUOTES, 'UTF-8', false)) . "'";
$text .= " url='" . $data['url'] . "'";
} }
if (!empty($data['title'])) {
$text .= " title='" . $data['title'] . "'";
} }
if (empty($data['text'])) { if (empty($data['text'])) {
@ -167,7 +162,7 @@ class PageInfo
} }
} }
$text .= ']' . $data['text'] . '[/attachment]'; $text .= ']' . str_replace(['[', ']'], ['[', ']'], $data['text']) . '[/attachment]';
$hashtags = ''; $hashtags = '';
if (!empty($data['keywords'])) { if (!empty($data['keywords'])) {
@ -192,7 +187,7 @@ class PageInfo
*/ */
public static function queryUrl(string $url, string $photo = '', bool $keywords = false, string $keyword_denylist = '') public static function queryUrl(string $url, string $photo = '', bool $keywords = false, string $keyword_denylist = '')
{ {
$data = ParseUrl::getSiteinfoCached($url, true); $data = ParseUrl::getSiteinfoCached($url);
if ($photo != '') { if ($photo != '') {
$data['images'][0]['src'] = $photo; $data['images'][0]['src'] = $photo;

6
src/Content/Text/BBCode.php
View file

@ -322,14 +322,14 @@ class BBCode
$data = ['url' => $url, 'type' => 'photo']; $data = ['url' => $url, 'type' => 'photo'];
} else { } else {
// Checking, if the link goes to a picture // Checking, if the link goes to a picture
$data = ParseUrl::getSiteinfoCached($pictures[0][1], true); $data = ParseUrl::getSiteinfoCached($pictures[0][1]);
} }
// Workaround: // Workaround:
// Sometimes photo posts to the own album are not detected at the start. // Sometimes photo posts to the own album are not detected at the start.
// So we seem to cannot use the cache for these cases. That's strange. // So we seem to cannot use the cache for these cases. That's strange.
if (($data['type'] != 'photo') && strstr($pictures[0][1], "/photos/")) { if (($data['type'] != 'photo') && strstr($pictures[0][1], "/photos/")) {
$data = ParseUrl::getSiteinfo($pictures[0][1], true); $data = ParseUrl::getSiteinfo($pictures[0][1]);
} }
if ($data['type'] == 'photo') { if ($data['type'] == 'photo') {
@ -416,7 +416,7 @@ class BBCode
$post['text'] = trim($body); $post['text'] = trim($body);
} }
} elseif (isset($post['url']) && ($post['type'] == 'video')) { } elseif (isset($post['url']) && ($post['type'] == 'video')) {
$data = ParseUrl::getSiteinfoCached($post['url'], true); $data = ParseUrl::getSiteinfoCached($post['url']);
if (isset($data['images'][0])) { if (isset($data['images'][0])) {
$post['image'] = $data['images'][0]['src']; $post['image'] = $data['images'][0]['src'];

2
src/Protocol/Feed.php
View file

@ -484,7 +484,7 @@ class Feed
$item["body"] = trim($item["title"]); $item["body"] = trim($item["title"]);
} }
$data = ParseUrl::getSiteinfoCached($item['plink'], true); $data = ParseUrl::getSiteinfoCached($item['plink']);
if (!empty($data['text']) && !empty($data['title']) && (mb_strlen($item['body']) < mb_strlen($data['text']))) { if (!empty($data['text']) && !empty($data['title']) && (mb_strlen($item['body']) < mb_strlen($data['text']))) {
// When the fetched page info text is longer than the body, we do try to enhance the body // When the fetched page info text is longer than the body, we do try to enhance the body
if (!empty($item['body']) && (strpos($data['title'], $item['body']) === false) && (strpos($data['text'], $item['body']) === false)) { if (!empty($item['body']) && (strpos($data['title'], $item['body']) === false) && (strpos($data['text'], $item['body']) === false)) {

590
src/Util/ParseUrl.php
View file

@ -75,8 +75,6 @@ class ParseUrl
* Search for chached embeddable data of an url otherwise fetch it * Search for chached embeddable data of an url otherwise fetch it
* *
* @param string $url The url of the page which should be scraped * @param string $url The url of the page which should be scraped
* @param bool $no_guessing If true the parse doens't search for
* preview pictures
* @param bool $do_oembed The false option is used by the function fetch_oembed() * @param bool $do_oembed The false option is used by the function fetch_oembed()
* to avoid endless loops * to avoid endless loops
* *
@ -85,7 +83,7 @@ class ParseUrl
* string 'type' => Content type * string 'type' => Content type
* string 'title' => (optional) The title of the content * string 'title' => (optional) The title of the content
* string 'text' => (optional) The description for the content * string 'text' => (optional) The description for the content
* string 'image' => (optional) A preview image of the content (only available if $no_geuessing = false) * string 'image' => (optional) A preview image of the content
* array 'images' => (optional) Array of preview pictures * array 'images' => (optional) Array of preview pictures
* string 'keywords' => (optional) The tags which belong to the content * string 'keywords' => (optional) The tags which belong to the content
* *
@ -93,7 +91,7 @@ class ParseUrl
* @see ParseUrl::getSiteinfo() for more information about scraping * @see ParseUrl::getSiteinfo() for more information about scraping
* embeddable content * embeddable content
*/ */
public static function getSiteinfoCached($url, $no_guessing = false, $do_oembed = true): array public static function getSiteinfoCached($url, $do_oembed = true): array
{ {
if (empty($url)) { if (empty($url)) {
return [ return [
@ -105,14 +103,14 @@ class ParseUrl
$urlHash = hash('sha256', $url); $urlHash = hash('sha256', $url);
$parsed_url = DBA::selectFirst('parsed_url', ['content'], $parsed_url = DBA::selectFirst('parsed_url', ['content'],
['url_hash' => $urlHash, 'guessing' => !$no_guessing, 'oembed' => $do_oembed] ['url_hash' => $urlHash, 'oembed' => $do_oembed]
); );
if (!empty($parsed_url['content'])) { if (!empty($parsed_url['content'])) {
$data = unserialize($parsed_url['content']); $data = unserialize($parsed_url['content']);
return $data; return $data;
} }
$data = self::getSiteinfo($url, $no_guessing, $do_oembed); $data = self::getSiteinfo($url, $do_oembed);
$expires = $data['expires']; $expires = $data['expires'];
@ -122,7 +120,6 @@ class ParseUrl
'parsed_url', 'parsed_url',
[ [
'url_hash' => $urlHash, 'url_hash' => $urlHash,
'guessing' => !$no_guessing,
'oembed' => $do_oembed, 'oembed' => $do_oembed,
'url' => $url, 'url' => $url,
'content' => serialize($data), 'content' => serialize($data),
@ -146,8 +143,6 @@ class ParseUrl
* \<meta name="description" content="An awesome description"\> * \<meta name="description" content="An awesome description"\>
* *
* @param string $url The url of the page which should be scraped * @param string $url The url of the page which should be scraped
* @param bool $no_guessing If true the parse doens't search for
* preview pictures
* @param bool $do_oembed The false option is used by the function fetch_oembed() * @param bool $do_oembed The false option is used by the function fetch_oembed()
* to avoid endless loops * to avoid endless loops
* @param int $count Internal counter to avoid endless loops * @param int $count Internal counter to avoid endless loops
@ -157,7 +152,7 @@ class ParseUrl
* string 'type' => Content type (error, link, photo, image, audio, video) * string 'type' => Content type (error, link, photo, image, audio, video)
* string 'title' => (optional) The title of the content * string 'title' => (optional) The title of the content
* string 'text' => (optional) The description for the content * string 'text' => (optional) The description for the content
* string 'image' => (optional) A preview image of the content (only available if $no_guessing = false) * string 'image' => (optional) A preview image of the content
* array 'images' => (optional) Array of preview pictures * array 'images' => (optional) Array of preview pictures
* string 'keywords' => (optional) The tags which belong to the content * string 'keywords' => (optional) The tags which belong to the content
* *
@ -175,7 +170,7 @@ class ParseUrl
* </body> * </body>
* @endverbatim * @endverbatim
*/ */
public static function getSiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) public static function getSiteinfo($url, $do_oembed = true, $count = 1)
{ {
if (empty($url)) { if (empty($url)) {
return [ return [
@ -241,7 +236,7 @@ class ParseUrl
$body = $curlResult->getBody(); $body = $curlResult->getBody();
if ($do_oembed) { if ($do_oembed) {
$oembed_data = OEmbed::fetchURL($url); $oembed_data = OEmbed::fetchURL($url, false, false);
if (!empty($oembed_data->type)) { if (!empty($oembed_data->type)) {
if (!in_array($oembed_data->type, ['error', 'rich', 'image', 'video', 'audio', ''])) { if (!in_array($oembed_data->type, ['error', 'rich', 'image', 'video', 'audio', ''])) {
@ -250,13 +245,25 @@ class ParseUrl
// See https://github.com/friendica/friendica/pull/5763#discussion_r217913178 // See https://github.com/friendica/friendica/pull/5763#discussion_r217913178
if ($siteinfo['type'] != 'photo') { if ($siteinfo['type'] != 'photo') {
if (isset($oembed_data->title)) { if (!empty($oembed_data->title)) {
$siteinfo['title'] = trim($oembed_data->title); $siteinfo['title'] = trim($oembed_data->title);
} }
if (isset($oembed_data->description)) { if (!empty($oembed_data->description)) {
$siteinfo['text'] = trim($oembed_data->description); $siteinfo['text'] = trim($oembed_data->description);
} }
if (isset($oembed_data->thumbnail_url)) { if (!empty($oembed_data->author_name)) {
$siteinfo['author_name'] = trim($oembed_data->author_name);
}
if (!empty($oembed_data->author_url)) {
$siteinfo['author_url'] = trim($oembed_data->author_url);
}
if (!empty($oembed_data->provider_name)) {
$siteinfo['publisher_name'] = trim($oembed_data->provider_name);
}
if (!empty($oembed_data->provider_url)) {
$siteinfo['publisher_url'] = trim($oembed_data->provider_url);
}
if (!empty($oembed_data->thumbnail_url)) {
$siteinfo['image'] = $oembed_data->thumbnail_url; $siteinfo['image'] = $oembed_data->thumbnail_url;
} }
} }
@ -300,7 +307,6 @@ class ParseUrl
@$doc->loadHTML($body); @$doc->loadHTML($body);
XML::deleteNode($doc, 'style'); XML::deleteNode($doc, 'style');
XML::deleteNode($doc, 'script');
XML::deleteNode($doc, 'option'); XML::deleteNode($doc, 'option');
XML::deleteNode($doc, 'h1'); XML::deleteNode($doc, 'h1');
XML::deleteNode($doc, 'h2'); XML::deleteNode($doc, 'h2');
@ -332,7 +338,7 @@ class ParseUrl
} }
} }
if ($content != '') { if ($content != '') {
$siteinfo = self::getSiteinfo($content, $no_guessing, $do_oembed, ++$count); $siteinfo = self::getSiteinfo($content, $do_oembed, ++$count);
return $siteinfo; return $siteinfo;
} }
} }
@ -392,6 +398,9 @@ class ParseUrl
case 'dc.description': case 'dc.description':
$siteinfo['text'] = trim($meta_tag['content']); $siteinfo['text'] = trim($meta_tag['content']);
break; break;
case 'dc.creator':
$siteinfo['publisher_name'] = trim($meta_tag['content']);
break;
case 'keywords': case 'keywords':
$keywords = explode(',', $meta_tag['content']); $keywords = explode(',', $meta_tag['content']);
break; break;
@ -426,12 +435,40 @@ class ParseUrl
case 'og:image': case 'og:image':
$siteinfo['image'] = $meta_tag['content']; $siteinfo['image'] = $meta_tag['content'];
break; break;
case 'og:image:url':
$siteinfo['image'] = $meta_tag['content'];
break;
case 'og:image:secure_url':
$siteinfo['image'] = $meta_tag['content'];
break;
case 'og:title': case 'og:title':
$siteinfo['title'] = trim($meta_tag['content']); $siteinfo['title'] = trim($meta_tag['content']);
break; break;
case 'og:description': case 'og:description':
$siteinfo['text'] = trim($meta_tag['content']); $siteinfo['text'] = trim($meta_tag['content']);
break; break;
case 'og:site_name':
$siteinfo['publisher_name'] = trim($meta_tag['content']);
break;
case 'twitter:description':
$siteinfo['text'] = trim($meta_tag['content']);
break;
case 'twitter:title':
$siteinfo['title'] = trim($meta_tag['content']);
break;
case 'twitter:image':
$siteinfo['image'] = $meta_tag['content'];
break;
}
}
}
$list = $xpath->query("//script[@type='application/ld+json']");
foreach ($list as $node) {
if (!empty($node->nodeValue)) {
$nodevalue = html_entity_decode($node->nodeValue, ENT_COMPAT, 'UTF-8');
if ($jsonld = json_decode($nodevalue, true)) {
$siteinfo = self::parseParts($siteinfo, $jsonld);
} }
} }
} }
@ -441,7 +478,7 @@ class ParseUrl
$siteinfo['type'] = 'link'; $siteinfo['type'] = 'link';
} }
if (!empty($siteinfo['image'])) { if (!empty($siteinfo['image']) && empty($siteinfo['images'])) {
$src = self::completeUrl($siteinfo['image'], $url); $src = self::completeUrl($siteinfo['image'], $url);
unset($siteinfo['image']); unset($siteinfo['image']);
@ -551,4 +588,521 @@ class ParseUrl
return($complete); return($complete);
} }
/**
* Parse the Json-Ld parts of a web page
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseParts(array $siteinfo, array $jsonld)
{
if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) {
foreach ($jsonld['@graph'] as $part) {
$siteinfo = self::parseParts($siteinfo, $part);
}
} elseif (!empty($jsonld['@type'])) {
$siteinfo = self::parseJsonLd($siteinfo, $jsonld);
} elseif (!empty($jsonld)) {
$keys = array_keys($jsonld);
$numeric_keys = true;
foreach ($keys as $key) {
if (!is_int($key)) {
$numeric_keys = false;
}
}
if ($numeric_keys) {
foreach ($jsonld as $part) {
$siteinfo = self::parseParts($siteinfo, $part);
}
}
}
return $siteinfo;
}
/**
* Improve the siteinfo with information from the provided JSON-LD information
* @see https://jsonld.com/
* @see https://schema.org/
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLd(array $siteinfo, array $jsonld)
{
$type = JsonLD::fetchElement($jsonld, '@type');
if (empty($type)) {
Logger::info('Empty type', ['url' => $siteinfo['url']]);
return $siteinfo;
}
// Silently ignore some types that aren't processed
if (in_array($type, ['SiteNavigationElement', 'JobPosting', 'CreativeWork', 'MusicAlbum',
'WPHeader', 'WPSideBar', 'WPFooter', 'LegalService', 'MusicRecording',
'ItemList', 'BreadcrumbList', 'Blog', 'Dataset', 'Product'])) {
return $siteinfo;
}
switch ($type) {
case 'Article':
case 'AdvertiserContentArticle':
case 'NewsArticle':
case 'Report':
case 'SatiricalArticle':
case 'ScholarlyArticle':
case 'SocialMediaPosting':
case 'TechArticle':
case 'ReportageNewsArticle':
case 'SocialMediaPosting':
case 'BlogPosting':
case 'LiveBlogPosting':
case 'DiscussionForumPosting':
return self::parseJsonLdArticle($siteinfo, $jsonld);
case 'WebPage':
case 'AboutPage':
case 'CheckoutPage':
case 'CollectionPage':
case 'ContactPage':
case 'FAQPage':
case 'ItemPage':
case 'MedicalWebPage':
case 'ProfilePage':
case 'QAPage':
case 'RealEstateListing':
case 'SearchResultsPage':
case 'MediaGallery':
case 'ImageGallery':
case 'VideoGallery':
case 'RadioEpisode':
case 'Event':
return self::parseJsonLdWebPage($siteinfo, $jsonld);
case 'WebSite':
return self::parseJsonLdWebSite($siteinfo, $jsonld);
case 'Organization':
case 'Airline':
case 'Consortium':
case 'Corporation':
case 'EducationalOrganization':
case 'FundingScheme':
case 'GovernmentOrganization':
case 'LibrarySystem':
case 'LocalBusiness':
case 'MedicalOrganization':
case 'NGO':
case 'NewsMediaOrganization':
case 'Project':
case 'SportsOrganization':
case 'WorkersUnion':
return self::parseJsonLdWebOrganization($siteinfo, $jsonld);
case 'Person':
case 'Patient':
case 'PerformingGroup':
case 'DanceGroup';
case 'MusicGroup':
case 'TheaterGroup':
return self::parseJsonLdWebPerson($siteinfo, $jsonld);
case 'AudioObject':
case 'Audio':
return self::parseJsonLdMediaObject($siteinfo, $jsonld, 'audio');
case 'VideoObject':
return self::parseJsonLdMediaObject($siteinfo, $jsonld, 'video');
case 'ImageObject':
return self::parseJsonLdMediaObject($siteinfo, $jsonld, 'images');
default:
Logger::info('Unknown type', ['type' => $type, 'url' => $siteinfo['url']]);
return $siteinfo;
}
}
/**
* Fetch author and publisher data
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdAuthor(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
if (!empty($jsonld['publisher']) && is_array($jsonld['publisher'])) {
$content = JsonLD::fetchElement($jsonld, 'publisher', 'name');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'publisher', 'sameAs');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'publisher', 'url');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$brand = JsonLD::fetchElement($jsonld, 'publisher', 'brand', '@type', 'Organization');
if (!empty($brand) && is_array($brand)) {
$content = JsonLD::fetchElement($brand, 'name');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
$content = JsonLD::fetchElement($brand, 'sameAs');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($brand, 'url');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($brand, 'logo', 'url');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_img'] = trim($content);
}
}
$logo = JsonLD::fetchElement($jsonld, 'publisher', 'logo');
if (!empty($logo) && is_array($logo)) {
$content = JsonLD::fetchElement($logo, 'url');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_img'] = trim($content);
}
}
} elseif (!empty($jsonld['publisher']) && is_string($jsonld['publisher'])) {
$jsonldinfo['publisher_name'] = trim($jsonld['publisher']);
}
if (!empty($jsonld['author']) && is_array($jsonld['author'])) {
$content = JsonLD::fetchElement($jsonld, 'author', 'name');
if (!empty($content) && is_string($content)) {
$jsonldinfo['author_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'author', 'sameAs');
if (!empty($content) && is_string($content)) {
$jsonldinfo['author_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'author', 'url');
if (!empty($content) && is_string($content)) {
$jsonldinfo['author_url'] = trim($content);
}
$logo = JsonLD::fetchElement($jsonld, 'author', 'logo');
if (!empty($logo) && is_array($logo)) {
$content = JsonLD::fetchElement($logo, 'url');
if (!empty($content) && is_string($content)) {
$jsonldinfo['author_img'] = trim($content);
}
}
} elseif (!empty($jsonld['author']) && is_string($jsonld['author'])) {
$jsonldinfo['author_name'] = trim($jsonld['author']);
}
Logger::info('Fetched Author information', ['fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Fetch data from the provided JSON-LD Article type
* @see https://schema.org/Article
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdArticle(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'headline');
if (!empty($content) && is_string($content)) {
$jsonldinfo['title'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'alternativeHeadline');
if (!empty($content) && is_string($content) && (($jsonldinfo['title'] ?? '') != trim($content))) {
$jsonldinfo['alternative_title'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content) && is_string($content)) {
$jsonldinfo['text'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
if (!empty($content)) {
$jsonldinfo['image'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject');
if (!empty($content) && is_string($content)) {
$jsonldinfo['image'] = trim($content);
}
if (!empty($jsonld['keywords']) && !is_array($jsonld['keywords'])) {
$content = JsonLD::fetchElement($jsonld, 'keywords');
if (!empty($content)) {
$siteinfo['keywords'] = [];
$keywords = explode(',', $content);
foreach ($keywords as $keyword) {
$siteinfo['keywords'][] = trim($keyword);
}
}
} else {
$content = JsonLD::fetchElementArray($jsonld, 'keywords');
if (!empty($content) && is_array($content)) {
$jsonldinfo['keywords'] = $content;
}
}
$jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
Logger::info('Fetched article information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Fetch data from the provided JSON-LD WebPage type
* @see https://schema.org/WebPage
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdWebPage(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content)) {
$jsonldinfo['title'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content)) {
$jsonldinfo['text'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'image');
if (!empty($content)) {
$jsonldinfo['image'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
if (!empty($content)) {
$jsonldinfo['image'] = trim($content);
}
$jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
Logger::info('Fetched WebPage information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Fetch data from the provided JSON-LD WebSite type
* @see https://schema.org/WebSite
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdWebSite(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content)) {
$jsonldinfo['publisher_description'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
if (!empty($content)) {
$jsonldinfo['image'] = trim($content);
}
$jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
Logger::info('Fetched WebSite information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Fetch data from the provided JSON-LD Organization type
* @see https://schema.org/Organization
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content)) {
$jsonldinfo['publisher_description'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'sameAs');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'logo', 'url', '@type', 'ImageObject');
if (!empty($content)) {
$jsonldinfo['publisher_img'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'brand', 'name', '@type', 'Organization');
if (!empty($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'brand', 'url', '@type', 'Organization');
if (!empty($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
Logger::info('Fetched Organization information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Fetch data from the provided JSON-LD Person type
* @see https://schema.org/Person
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld)
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content)) {
$jsonldinfo['author_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content)) {
$jsonldinfo['author_description'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'sameAs');
if (!empty($content) && is_string($content)) {
$jsonldinfo['author_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
$jsonldinfo['author_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject');
if (!empty($content)) {
$jsonldinfo['author_img'] = trim($content);
}
Logger::info('Fetched Person information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
return array_merge($siteinfo, $jsonldinfo);
}
/**
* Fetch data from the provided JSON-LD MediaObject type
* @see https://schema.org/MediaObject
*
* @param array $siteinfo
* @param array $jsonld
* @return array siteinfo
*/
private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name)
{
$media = [];
$content = JsonLD::fetchElement($jsonld, 'caption');
if (!empty($content)) {
$media['caption'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
$media['src'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
if (!empty($content)) {
$media['description'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'name');
if (!empty($content) && (($media['description'] ?? '') != trim($content))) {
$media['name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'contentUrl');
if (!empty($content)) {
$media['content'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'embedUrl');
if (!empty($content)) {
$media['embed'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'height');
if (!empty($content)) {
$media['height'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'width');
if (!empty($content)) {
$media['width'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
if (!empty($content)) {
$media['preview'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'image');
if (!empty($content)) {
$media['image'] = trim($content);
}
Logger::info('Fetched Media information', ['url' => $siteinfo['url'], 'fetched' => $media]);
$siteinfo[$name][] = $media;
return $siteinfo;
}
} }