diff --git a/mod/ping.php b/mod/ping.php index d1983e803..848f2f0ec 100644 --- a/mod/ping.php +++ b/mod/ping.php @@ -136,13 +136,9 @@ function ping_init(App $a) $notifs = ping_get_notifications(local_user()); - $condition = ["`unseen` AND `uid` = ? AND `contact-id` != ? AND (`vid` != ? OR `vid` IS NULL)", - local_user(), local_user(), Verb::getID(Activity::FOLLOW)]; - $fields = ['id', 'parent', 'verb', 'author-name', 'unseen', 'author-link', 'author-avatar', 'contact-avatar', - 'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid', 'wall', 'activity']; - $params = ['order' => ['received' => true]]; - $items = Item::selectForUser(local_user(), $fields, $condition, $params); - + $condition = ["`unseen` AND `uid` = ? AND NOT `origin` AND (`vid` != ? OR `vid` IS NULL)", + local_user(), Verb::getID(Activity::FOLLOW)]; + $items = Item::selectForUser(local_user(), ['wall', 'uid', 'uri-id'], $condition); if (DBA::isResult($items)) { $items_unseen = Item::inArray($items); $arr = ['items' => $items_unseen]; @@ -156,6 +152,7 @@ function ping_init(App $a) } } } + DBA::close($items); if ($network_count) { // Find out how unseen network posts are spread across groups diff --git a/src/Content/PageInfo.php b/src/Content/PageInfo.php index 642c57938..e4d9207c8 100644 --- a/src/Content/PageInfo.php +++ b/src/Content/PageInfo.php @@ -40,7 +40,7 @@ class PageInfo * @return string * @throws HTTPException\InternalServerErrorException */ - public static function appendToBody(string $body, bool $searchNakedUrls = false, bool $no_photos = false) + public static function searchAndAppendToBody(string $body, bool $searchNakedUrls = false, bool $no_photos = false) { Logger::info('add_page_info_to_body: fetch page info for body', ['body' => $body]); @@ -49,14 +49,34 @@ class PageInfo return $body; } - $footer = self::getFooterFromUrl($url, $no_photos); - if (!$footer) { + $data = self::queryUrl($url); + if (!$data) { return $body; } - $body = self::stripTrailingUrlFromBody($body, $url); + return self::appendDataToBody($body, $data, $no_photos); + } - $body .= "\n" . $footer; + /** + * @param string $body + * @param array $data + * @param bool $no_photos + * @return string + * @throws HTTPException\InternalServerErrorException + */ + public static function appendDataToBody(string $body, array $data, bool $no_photos = false) + { + // Only one [attachment] tag per body is allowed + $existingAttachmentPos = strpos($body, '[attachment'); + if ($existingAttachmentPos !== false) { + $linkTitle = $data['title'] ?: $data['url']; + // Additional link attachments are prepended before the existing [attachment] tag + $body = substr_replace($body, "\n[bookmark=" . $data['url'] . ']' . $linkTitle . "[/bookmark]\n", $existingAttachmentPos, 0); + } else { + $footer = PageInfo::getFooterFromData($data, $no_photos); + $body = self::stripTrailingUrlFromBody($body, $data['url']); + $body .= "\n" . $footer; + } return $body; } @@ -252,22 +272,35 @@ class PageInfo /** * Remove the provided URL from the body if it is at the end of it. - * Keep the link label if it isn't the full URL. + * Keep the link label if it isn't the full URL or a shortened version of it. * * @param string $body * @param string $url - * @return string|string[]|null + * @return string */ protected static function stripTrailingUrlFromBody(string $body, string $url) { $quotedUrl = preg_quote($url, '#'); - $body = preg_replace("#(?: + $body = preg_replace_callback("#(?: \[url]$quotedUrl\[/url]| \[url=$quotedUrl]$quotedUrl\[/url]| \[url=$quotedUrl]([^[]*?)\[/url]| $quotedUrl - )$#isx", '$1', $body); + )$#isx", function ($match) use ($url) { + // Stripping URLs with no label + if (!isset($match[1])) { + return ''; + } - return $body; + // Stripping link labels that include a shortened version of the URL + if (strpos($url, trim($match[1], '.…')) !== false) { + return ''; + } + + // Keep all other labels + return $match[1]; + }, $body); + + return rtrim($body); } } diff --git a/src/Database/Database.php b/src/Database/Database.php index eaf490050..5ef481556 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -343,7 +343,7 @@ class Database $row['key'] . "\t" . $row['rows'] . "\t" . $row['Extra'] . "\t" . basename($backtrace[1]["file"]) . "\t" . $backtrace[1]["line"] . "\t" . $backtrace[2]["function"] . "\t" . - substr($query, 0, 2000) . "\n", FILE_APPEND); + substr($query, 0, 4000) . "\n", FILE_APPEND); } } } @@ -712,7 +712,7 @@ class Database @file_put_contents($this->configCache->get('system', 'db_log'), DateTimeFormat::utcNow() . "\t" . $duration . "\t" . basename($backtrace[1]["file"]) . "\t" . $backtrace[1]["line"] . "\t" . $backtrace[2]["function"] . "\t" . - substr($this->replaceParameters($sql, $args), 0, 2000) . "\n", FILE_APPEND); + substr($this->replaceParameters($sql, $args), 0, 4000) . "\n", FILE_APPEND); } } return $retval; diff --git a/src/Model/Item.php b/src/Model/Item.php index 332c734fa..c75286b25 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -2093,7 +2093,7 @@ class Item DBA::close($contacts); if (!empty($owner['alias'])) { - $condition = ['url' => $owner['alias'], 'rel' => [Contact::SHARING, Contact::FRIEND]]; + $condition = ['nurl' => Strings::normaliseLink($owner['alias']), 'rel' => [Contact::SHARING, Contact::FRIEND]]; $contacts = DBA::select('contact', ['uid'], $condition); while ($contact = DBA::fetch($contacts)) { if ($contact['uid'] == 0) { diff --git a/src/Module/Debug/Babel.php b/src/Module/Debug/Babel.php index 2954bc010..5b89c5301 100644 --- a/src/Module/Debug/Babel.php +++ b/src/Module/Debug/Babel.php @@ -24,9 +24,12 @@ namespace Friendica\Module\Debug; use Friendica\BaseModule; use Friendica\Content\PageInfo; use Friendica\Content\Text; +use Friendica\Core\Protocol; use Friendica\Core\Renderer; use Friendica\DI; +use Friendica\Model\Conversation; use Friendica\Model\Item; +use Friendica\Protocol\Activity; use Friendica\Model\Tag; use Friendica\Util\XML; @@ -115,7 +118,7 @@ class Babel extends BaseModule 'content' => visible_whitespace(var_export($tags, true)), ]; - $body2 = PageInfo::appendToBody($bbcode, true); + $body2 = PageInfo::searchAndAppendToBody($bbcode, true); $results[] = [ 'title' => DI::l10n()->t('PageInfo::appendToBody'), 'content' => visible_whitespace($body2) @@ -215,6 +218,60 @@ class Babel extends BaseModule 'title' => DI::l10n()->t('HTML::toPlaintext (compact)'), 'content' => visible_whitespace($text), ]; + break; + case 'twitter': + $json = trim($_REQUEST['text']); + + $status = json_decode($json); + + $results[] = [ + 'title' => DI::l10n()->t('Decoded post'), + 'content' => visible_whitespace(var_export($status, true)), + ]; + + $postarray = []; + $postarray['object-type'] = Activity\ObjectType::NOTE; + + if (!empty($status->full_text)) { + $postarray['body'] = $status->full_text; + } else { + $postarray['body'] = $status->text; + } + + // When the post contains links then use the correct object type + if (count($status->entities->urls) > 0) { + $postarray['object-type'] = Activity\ObjectType::BOOKMARK; + } + + if (file_exists('addon/twitter/twitter.php')) { + require_once 'addon/twitter/twitter.php'; + + $picture = \twitter_media_entities($status, $postarray); + + $results[] = [ + 'title' => DI::l10n()->t('Post array before expand entities'), + 'content' => visible_whitespace(var_export($postarray, true)), + ]; + + $converted = \twitter_expand_entities($postarray['body'], $status, $picture); + + $results[] = [ + 'title' => DI::l10n()->t('Post converted'), + 'content' => visible_whitespace(var_export($converted, true)), + ]; + + $results[] = [ + 'title' => DI::l10n()->t('Converted body'), + 'content' => visible_whitespace($converted['body']), + ]; + } else { + $results[] = [ + 'title' => DI::l10n()->t('Error'), + 'content' => DI::l10n()->t('Twitter addon is absent from the addon/ folder.'), + ]; + } + + break; } } @@ -225,6 +282,8 @@ class Babel extends BaseModule '$type_diaspora' => ['type', DI::l10n()->t('Diaspora'), 'diaspora', '', (($_REQUEST['type'] ?? '') ?: 'bbcode') == 'diaspora'], '$type_markdown' => ['type', DI::l10n()->t('Markdown'), 'markdown', '', (($_REQUEST['type'] ?? '') ?: 'bbcode') == 'markdown'], '$type_html' => ['type', DI::l10n()->t('HTML'), 'html', '', (($_REQUEST['type'] ?? '') ?: 'bbcode') == 'html'], + '$flag_twitter' => file_exists('addon/twitter/twitter.php'), + '$type_twitter' => ['type', DI::l10n()->t('Twitter Source'), 'twitter', '', (($_REQUEST['type'] ?? '') ?: 'bbcode') == 'twitter'], '$results' => $results ]); diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 745a56c2a..e4cef1704 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -21,6 +21,7 @@ namespace Friendica\Protocol\ActivityPub; +use Friendica\Content\PageInfo; use Friendica\Content\Text\BBCode; use Friendica\Content\Text\HTML; use Friendica\Core\Logger; @@ -96,18 +97,16 @@ class Processor foreach ($activity['attachments'] as $attach) { switch ($attach['type']) { case 'link': - // Only one [attachment] tag is allowed - $existingAttachmentPos = strpos($item['body'], '[attachment'); - if ($existingAttachmentPos !== false) { - $linkTitle = $attach['title'] ?: $attach['url']; - // Additional link attachments are prepended before the existing [attachment] tag - $item['body'] = substr_replace($item['body'], "\n[bookmark=" . $attach['url'] . ']' . $linkTitle . "[/bookmark]\n", $existingAttachmentPos, 0); - } else { - // Strip the link preview URL from the end of the body if any - $quotedUrl = preg_quote($attach['url'], '#'); - $item['body'] = preg_replace("#\s*(?:\[bookmark={$quotedUrl}].+?\[/bookmark]|\[url={$quotedUrl}].+?\[/url]|\[url]{$quotedUrl}\[/url]|{$quotedUrl})\s*$#", '', $item['body']); - $item['body'] .= "\n[attachment type='link' url='" . $attach['url'] . "' title='" . htmlspecialchars($attach['title'] ?? '', ENT_QUOTES) . "' image='" . ($attach['image'] ?? '') . "']" . ($attach['desc'] ?? '') . '[/attachment]'; - } + $data = [ + 'url' => $attach['url'], + 'type' => $attach['type'], + 'title' => $attach['title'] ?? '', + 'text' => $attach['desc'] ?? '', + 'image' => $attach['image'] ?? '', + 'images' => [], + 'keywords' => [], + ]; + $item['body'] = PageInfo::appendDataToBody($item['body'], $data); break; default: $filetype = strtolower(substr($attach['mediaType'], 0, strpos($attach['mediaType'], '/'))); diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 0dfcf6f77..bd99b361e 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -2622,7 +2622,7 @@ class Diaspora $item["body"] = self::replacePeopleGuid($item["body"], $item["author-link"]); // Add OEmbed and other information to the body - $item["body"] = PageInfo::appendToBody($item["body"], false, true); + $item["body"] = PageInfo::searchAndAppendToBody($item["body"], false, true); return $item; } else { @@ -2986,7 +2986,7 @@ class Diaspora // Add OEmbed and other information to the body if (!self::isHubzilla($contact["url"])) { - $body = PageInfo::appendToBody($body, false, true); + $body = PageInfo::searchAndAppendToBody($body, false, true); } } diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index fedf0f253..9a52476b5 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -698,7 +698,7 @@ class OStatus // Only add additional data when there is no picture in the post if (!strstr($item["body"], '[/img]')) { - $item["body"] = PageInfo::appendToBody($item["body"]); + $item["body"] = PageInfo::searchAndAppendToBody($item["body"]); } Tag::storeFromBody($item['uri-id'], $item['body']); diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index 62b5d007d..b6d172a3a 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -55,14 +55,13 @@ class ParseUrl * to avoid endless loops * * @return array which contains needed data for embedding - * string 'url' => The url of the parsed page - * string 'type' => Content type - * string 'title' => The title of the content - * string 'text' => The description for the content - * string 'image' => A preview image of the content (only available - * if $no_geuessing = false - * array'images' = Array of preview pictures - * string 'keywords' => The tags which belong to the content + * string 'url' => The url of the parsed page + * string 'type' => Content type + * string 'title' => (optional) The title of the content + * string 'text' => (optional) The description for the content + * string 'image' => (optional) A preview image of the content (only available if $no_geuessing = false) + * array 'images' => (optional) Array of preview pictures + * string 'keywords' => (optional) The tags which belong to the content * * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @see ParseUrl::getSiteinfo() for more information about scraping @@ -115,14 +114,13 @@ class ParseUrl * @param int $count Internal counter to avoid endless loops * * @return array which contains needed data for embedding - * string 'url' => The url of the parsed page - * string 'type' => Content type - * string 'title' => The title of the content - * string 'text' => The description for the content - * string 'image' => A preview image of the content (only available - * if $no_geuessing = false - * array'images' = Array of preview pictures - * string 'keywords' => The tags which belong to the content + * string 'url' => The url of the parsed page + * string 'type' => Content type + * string 'title' => (optional) The title of the content + * string 'text' => (optional) The description for the content + * string 'image' => (optional) A preview image of the content (only available if $no_guessing = false) + * array 'images' => (optional) Array of preview pictures + * string 'keywords' => (optional) The tags which belong to the content * * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @todo https://developers.google.com/+/plugins/snippet/ @@ -140,28 +138,27 @@ class ParseUrl */ public static function getSiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) { - $siteinfo = []; - // Check if the URL does contain a scheme $scheme = parse_url($url, PHP_URL_SCHEME); if ($scheme == '') { - $url = 'http://' . trim($url, '/'); + $url = 'http://' . ltrim($url, '/'); } + $url = trim($url, "'\""); + + $url = Network::stripTrackingQueryParams($url); + + $siteinfo = [ + 'url' => $url, + 'type' => 'link', + ]; + if ($count > 10) { Logger::log('Endless loop detected for ' . $url, Logger::DEBUG); return $siteinfo; } - $url = trim($url, "'"); - $url = trim($url, '"'); - - $url = Network::stripTrackingQueryParams($url); - - $siteinfo['url'] = $url; - $siteinfo['type'] = 'link'; - $curlResult = Network::curl($url); if (!$curlResult->isSuccess()) { return $siteinfo; diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index a47193334..6749b9648 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -160,7 +160,6 @@ class Cron $condition = ["`network` IN (?, ?, ?, ?) AND `uid` = ? AND NOT `self` AND `last-update` < ?", Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, 0, $last_updated]; - $total = DBA::count('contact', $condition); $oldest_date = ''; $oldest_id = ''; $contacts = DBA::select('contact', ['id', 'last-update'], $condition, ['limit' => 100, 'order' => ['last-update']]); @@ -172,7 +171,7 @@ class Cron Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id'], 'force'); ++$count; } - Logger::info('Initiated update for public contacts', ['interval' => $count, 'total' => $total, 'id' => $oldest_id, 'oldest' => $oldest_date]); + Logger::info('Initiated update for public contacts', ['interval' => $count, 'id' => $oldest_id, 'oldest' => $oldest_date]); DBA::close($contacts); } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 8908ba13a..7433531fe 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -54,7 +54,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1355); + define('DB_UPDATE_VERSION', 1356); } return [ @@ -200,6 +200,7 @@ return [ "attag_uid" => ["attag(32)", "uid"], "dfrn-id" => ["dfrn-id(64)"], "issued-id" => ["issued-id(64)"], + "network_uid_lastupdate" => ["network", "uid", "last-update"], "gsid" => ["gsid"] ] ], @@ -321,6 +322,7 @@ return [ "addr" => ["addr(32)"], "alias" => ["alias(190)"], "followers" => ["followers(190)"], + "baseurl" => ["baseurl(190)"], "gsid" => ["gsid"] ] ], diff --git a/tests/src/Content/PageInfoTest.php b/tests/src/Content/PageInfoTest.php index 6f9641564..3604348ff 100644 --- a/tests/src/Content/PageInfoTest.php +++ b/tests/src/Content/PageInfoTest.php @@ -108,6 +108,21 @@ class PageInfoTest extends MockedTest 'body' => '[url=https://example.com]link label[/url]', 'url' => 'https://example.com', ], + 'task-8797-shortened-link-label' => [ + 'expected' => 'content', + 'body' => 'content [url=https://example.com/page]example.com/[/url]', + 'url' => 'https://example.com/page', + ], + 'task-8797-shortened-link-label-ellipsis' => [ + 'expected' => 'content', + 'body' => 'content [url=https://example.com/page]example.com…[/url]', + 'url' => 'https://example.com/page', + ], + 'task-8797-shortened-link-label-dots' => [ + 'expected' => 'content', + 'body' => 'content [url=https://example.com/page]example.com...[/url]', + 'url' => 'https://example.com/page', + ], ]; } diff --git a/view/templates/babel.tpl b/view/templates/babel.tpl index 4dc50083d..57d17fea9 100644 --- a/view/templates/babel.tpl +++ b/view/templates/babel.tpl @@ -9,6 +9,9 @@ {{include file="field_radio.tpl" field=$type_diaspora}} {{include file="field_radio.tpl" field=$type_markdown}} {{include file="field_radio.tpl" field=$type_html}} + {{if $flag_twitter}} + {{include file="field_radio.tpl" field=$type_twitter}} + {{/if}}