From 661dd43b0ddac517ac254b68a2ab7a58b3c48153 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 13 Apr 2020 19:24:22 +0000 Subject: [PATCH 01/43] Term constants updated --- boot.php | 23 ----------------------- include/api.php | 3 ++- mod/network.php | 2 +- mod/tagger.php | 7 ++++--- src/Content/Widget/TagCloud.php | 7 ++++--- src/Model/FileTag.php | 5 +++-- src/Model/Term.php | 3 --- src/Model/UserItem.php | 3 ++- src/Module/Hashtag.php | 3 ++- src/Module/Profile/Status.php | 5 +++-- src/Protocol/ActivityPub/Transmitter.php | 2 +- src/Protocol/DFRN.php | 9 +++------ src/Protocol/Diaspora.php | 3 ++- src/Worker/TagUpdate.php | 7 ++++--- 14 files changed, 31 insertions(+), 51 deletions(-) diff --git a/boot.php b/boot.php index 348e1b2c41..0637ff3fea 100644 --- a/boot.php +++ b/boot.php @@ -178,29 +178,6 @@ define('NOTIFY_SHARE', Notify\Type::SHARE); define('NOTIFY_SYSTEM', Notify\Type::SYSTEM); /* @}*/ - -/** @deprecated since 2019.03, use Term::UNKNOWN instead */ -define('TERM_UNKNOWN', Term::UNKNOWN); -/** @deprecated since 2019.03, use Term::HASHTAG instead */ -define('TERM_HASHTAG', Term::HASHTAG); -/** @deprecated since 2019.03, use Term::MENTION instead */ -define('TERM_MENTION', Term::MENTION); -/** @deprecated since 2019.03, use Term::CATEGORY instead */ -define('TERM_CATEGORY', Term::CATEGORY); -/** @deprecated since 2019.03, use Term::PCATEGORY instead */ -define('TERM_PCATEGORY', Term::PCATEGORY); -/** @deprecated since 2019.03, use Term::FILE instead */ -define('TERM_FILE', Term::FILE); -/** @deprecated since 2019.03, use Term::SAVEDSEARCH instead */ -define('TERM_SAVEDSEARCH', Term::SAVEDSEARCH); -/** @deprecated since 2019.03, use Term::CONVERSATION instead */ -define('TERM_CONVERSATION', Term::CONVERSATION); - -/** @deprecated since 2019.03, use Term::OBJECT_TYPE_POST instead */ -define('TERM_OBJ_POST', Term::OBJECT_TYPE_POST); -/** @deprecated since 2019.03, use Term::OBJECT_TYPE_PHOTO instead */ -define('TERM_OBJ_PHOTO', Term::OBJECT_TYPE_PHOTO); - /** * @name Gravity * diff --git a/include/api.php b/include/api.php index 8291d18926..75e66349eb 100644 --- a/include/api.php +++ b/include/api.php @@ -41,6 +41,7 @@ use Friendica\Model\Item; use Friendica\Model\Mail; use Friendica\Model\Notify; use Friendica\Model\Photo; +use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Model\UserItem; use Friendica\Network\FKOAuth1; @@ -1541,7 +1542,7 @@ function api_search($type) $condition = ["`oid` > ? AND (`uid` = 0 OR (`uid` = ? AND NOT `global`)) AND `otype` = ? AND `type` = ? AND `term` = ?", - $since_id, local_user(), TERM_OBJ_POST, TERM_HASHTAG, $searchTerm]; + $since_id, local_user(), Term::OBJECT_TYPE_POST, Term::HASHTAG, $searchTerm]; if ($max_id > 0) { $condition[0] .= ' AND `oid` <= ?'; $condition[] = $max_id; diff --git a/mod/network.php b/mod/network.php index d252175df1..6c02c4f843 100644 --- a/mod/network.php +++ b/mod/network.php @@ -793,7 +793,7 @@ function networkThreadedView(App $a, $update, $parent) STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = `item`.`author-id` WHERE `item`.`uid` = 0 AND `item`.$ordering < ? AND `item`.$ordering > ? AND `item`.`gravity` = ? AND NOT `author`.`hidden` AND NOT `author`.`blocked`" . $sql_tag_nets, - local_user(), TERM_OBJ_POST, TERM_HASHTAG, + local_user(), Term::OBJECT_TYPE_POST, Term::HASHTAG, $top_limit, $bottom_limit, GRAVITY_PARENT); $data = DBA::toArray($items); diff --git a/mod/tagger.php b/mod/tagger.php index a6f35cea12..e3ae6b9b5c 100644 --- a/mod/tagger.php +++ b/mod/tagger.php @@ -28,6 +28,7 @@ use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; +use Friendica\Model\Term; use Friendica\Protocol\Activity; use Friendica\Util\Strings; use Friendica\Util\XML; @@ -168,7 +169,7 @@ EOT; Item::update(['visible' => true], ['id' => $item['id']]); } - $term_objtype = ($item['resource-id'] ? TERM_OBJ_PHOTO : TERM_OBJ_POST); + $term_objtype = ($item['resource-id'] ? Term::OBJECT_TYPE_PHOTO : Term::OBJECT_TYPE_POST); $t = q("SELECT count(tid) as tcount FROM term WHERE oid=%d AND term='%s'", intval($item['id']), @@ -179,7 +180,7 @@ EOT; q("INSERT INTO term (oid, otype, type, term, url, uid) VALUE (%d, %d, %d, '%s', '%s', %d)", intval($item['id']), $term_objtype, - TERM_HASHTAG, + Term::HASHTAG, DBA::escape($term), '', intval($owner_uid) @@ -201,7 +202,7 @@ EOT; q("INSERT INTO term (`oid`, `otype`, `type`, `term`, `url`, `uid`) VALUE (%d, %d, %d, '%s', '%s', %d)", intval($original_item['id']), $term_objtype, - TERM_HASHTAG, + Term::HASHTAG, DBA::escape($term), '', intval($owner_uid) diff --git a/src/Content/Widget/TagCloud.php b/src/Content/Widget/TagCloud.php index 006aef9161..5bdf7d8346 100644 --- a/src/Content/Widget/TagCloud.php +++ b/src/Content/Widget/TagCloud.php @@ -25,6 +25,7 @@ use Friendica\Core\Renderer; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; +use Friendica\Model\Term; /** * TagCloud widget @@ -45,7 +46,7 @@ class TagCloud * @return string HTML formatted output. * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function getHTML($uid, $count = 0, $owner_id = 0, $flags = '', $type = TERM_HASHTAG) + public static function getHTML($uid, $count = 0, $owner_id = 0, $flags = '', $type = Term::HASHTAG) { $o = ''; $r = self::tagadelic($uid, $count, $owner_id, $flags, $type); @@ -84,7 +85,7 @@ class TagCloud * @return array Alphabetical sorted array of used tags of an user. * @throws \Exception */ - private static function tagadelic($uid, $count = 0, $owner_id = 0, $flags = '', $type = TERM_HASHTAG) + private static function tagadelic($uid, $count = 0, $owner_id = 0, $flags = '', $type = Term::HASHTAG) { $sql_options = Item::getPermissionsSQLByUserId($uid); $limit = $count ? sprintf('LIMIT %d', intval($count)) : ''; @@ -109,7 +110,7 @@ class TagCloud GROUP BY `term` ORDER BY `total` DESC $limit", $uid, $type, - TERM_OBJ_POST + Term::OBJECT_TYPE_POST ); if (!DBA::isResult($tag_stmt)) { return []; diff --git a/src/Model/FileTag.php b/src/Model/FileTag.php index 02eee8aeda..3539f2b96d 100644 --- a/src/Model/FileTag.php +++ b/src/Model/FileTag.php @@ -23,6 +23,7 @@ namespace Friendica\Model; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\Term; /** * This class handles FileTag related functions @@ -195,11 +196,11 @@ class FileTag if ($type == 'file') { $lbracket = '['; $rbracket = ']'; - $termtype = TERM_FILE; + $termtype = Term::FILE; } else { $lbracket = '<'; $rbracket = '>'; - $termtype = TERM_CATEGORY; + $termtype = Term::CATEGORY; } $filetags_updated = $saved; diff --git a/src/Model/Term.php b/src/Model/Term.php index 84cb1bea7a..928c35f207 100644 --- a/src/Model/Term.php +++ b/src/Model/Term.php @@ -40,10 +40,7 @@ class Term const HASHTAG = 1; const MENTION = 2; const CATEGORY = 3; - const PCATEGORY = 4; const FILE = 5; - const SAVEDSEARCH = 6; - const CONVERSATION = 7; /** * An implicit mention is a mention in a comment body that is redundant with the threading information. */ diff --git a/src/Model/UserItem.php b/src/Model/UserItem.php index 0b0a4d2033..d38ed6d55e 100644 --- a/src/Model/UserItem.php +++ b/src/Model/UserItem.php @@ -26,6 +26,7 @@ use Friendica\Core\Hook; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Util\Strings; +use Friendica\Model\Term; class UserItem { @@ -206,7 +207,7 @@ class UserItem } // Or the contact is a mentioned forum - $tags = DBA::select('term', ['url'], ['otype' => TERM_OBJ_POST, 'oid' => $item['id'], 'type' => TERM_MENTION, 'uid' => $uid]); + $tags = DBA::select('term', ['url'], ['otype' => term::OBJECT_TYPE_POST, 'oid' => $item['id'], 'type' => Term::MENTION, 'uid' => $uid]); while ($tag = DBA::fetch($tags)) { $condition = ['nurl' => Strings::normaliseLink($tag['url']), 'uid' => $uid, 'notify_new_posts' => true, 'contact-type' => Contact::TYPE_COMMUNITY]; if (DBA::exists('contact', $condition)) { diff --git a/src/Module/Hashtag.php b/src/Module/Hashtag.php index 50719774fb..06c6374e34 100644 --- a/src/Module/Hashtag.php +++ b/src/Module/Hashtag.php @@ -25,6 +25,7 @@ use Friendica\BaseModule; use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\Util\Strings; +use Friendica\Model\Term; /** * Hashtag module. @@ -43,7 +44,7 @@ class Hashtag extends BaseModule $taglist = DBA::p("SELECT DISTINCT(`term`) FROM `term` WHERE `term` LIKE ? AND `type` = ? ORDER BY `term`", $t . '%', - intval(TERM_HASHTAG) + intval(Term::HASHTAG) ); while ($tag = DBA::fetch($taglist)) { $result[] = ['text' => $tag['term']]; diff --git a/src/Module/Profile/Status.php b/src/Module/Profile/Status.php index 338cf6ef4e..bb19be2b9c 100644 --- a/src/Module/Profile/Status.php +++ b/src/Module/Profile/Status.php @@ -31,6 +31,7 @@ use Friendica\DI; use Friendica\Model\Item; use Friendica\Model\Profile as ProfileModel; use Friendica\Model\User; +use Friendica\Model\Term; use Friendica\Module\BaseProfile; use Friendica\Module\Security\Login; use Friendica\Util\DateTimeFormat; @@ -142,12 +143,12 @@ class Status extends BaseProfile if (!empty($category)) { $sql_post_table = sprintf("INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ", - DBA::escape(Strings::protectSprintf($category)), intval(TERM_OBJ_POST), intval(TERM_CATEGORY), intval($a->profile['uid'])); + DBA::escape(Strings::protectSprintf($category)), intval(Term::OBJECT_TYPE_POST), intval(Term::CATEGORY), intval($a->profile['uid'])); } if (!empty($hashtags)) { $sql_post_table .= sprintf("INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ", - DBA::escape(Strings::protectSprintf($hashtags)), intval(TERM_OBJ_POST), intval(TERM_HASHTAG), intval($a->profile['uid'])); + DBA::escape(Strings::protectSprintf($hashtags)), intval(Term::OBJECT_TYPE_POST), intval(Term::HASHTAG), intval($a->profile['uid'])); } if (!empty($datequery)) { diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index ecda2dc683..e35d833615 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -1218,7 +1218,7 @@ class Transmitter */ private static function isSensitive($item_id) { - $condition = ['otype' => TERM_OBJ_POST, 'oid' => $item_id, 'type' => TERM_HASHTAG, 'term' => 'nsfw']; + $condition = ['otype' => Term::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => Term::HASHTAG, 'term' => 'nsfw']; return DBA::exists('term', $condition); } diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 4c71de4d57..6fb63c3d46 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -24,9 +24,7 @@ namespace Friendica\Protocol; use DOMDocument; use DOMXPath; use Friendica\App\BaseURL; -use Friendica\Content\OEmbed; use Friendica\Content\Text\BBCode; -use Friendica\Content\Text\HTML; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\Protocol; @@ -41,6 +39,7 @@ use Friendica\Model\Mail; use Friendica\Model\Notify\Type; use Friendica\Model\PermissionSet; use Friendica\Model\Profile; +use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Util\Crypto; @@ -49,8 +48,6 @@ use Friendica\Util\Images; use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; -use HTMLPurifier; -use HTMLPurifier_Config; /** * This class contain functions to create and send DFRN XML files @@ -252,8 +249,8 @@ class DFRN $sql_post_table = sprintf( "INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ", DBA::escape(Strings::protectSprintf($category)), - intval(TERM_OBJ_POST), - intval(TERM_CATEGORY), + intval(Term::OBJECT_TYPE_POST), + intval(Term::CATEGORY), intval($owner_id) ); } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 828333b41f..8bb407ebd8 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -38,6 +38,7 @@ use Friendica\Model\Item; use Friendica\Model\ItemDeliveryData; use Friendica\Model\Mail; use Friendica\Model\Profile; +use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Util\Crypto; @@ -123,7 +124,7 @@ class Diaspora } // All tags of the current post - $condition = ['otype' => TERM_OBJ_POST, 'type' => TERM_HASHTAG, 'oid' => $parent['parent']]; + $condition = ['otype' => Term::OBJECT_TYPE_POST, 'type' => Term::HASHTAG, 'oid' => $parent['parent']]; $tags = DBA::select('term', ['term'], $condition); $taglist = []; while ($tag = DBA::fetch($tags)) { diff --git a/src/Worker/TagUpdate.php b/src/Worker/TagUpdate.php index d7ad4462c3..1b4ba8d79f 100644 --- a/src/Worker/TagUpdate.php +++ b/src/Worker/TagUpdate.php @@ -23,6 +23,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Database\DBA; +use Friendica\Model\Term; class TagUpdate { @@ -35,14 +36,14 @@ class TagUpdate if ($message['uid'] == 0) { $global = true; - DBA::update('term', ['global' => true], ['otype' => TERM_OBJ_POST, 'guid' => $message['guid']]); + DBA::update('term', ['global' => true], ['otype' => Term::OBJECT_TYPE_POST, 'guid' => $message['guid']]); } else { - $global = (DBA::count('term', ['uid' => 0, 'otype' => TERM_OBJ_POST, 'guid' => $message['guid']]) > 0); + $global = (DBA::count('term', ['uid' => 0, 'otype' => Term::OBJECT_TYPE_POST, 'guid' => $message['guid']]) > 0); } $fields = ['guid' => $message['guid'], 'created' => $message['created'], 'received' => $message['received'], 'global' => $global]; - DBA::update('term', $fields, ['otype' => TERM_OBJ_POST, 'oid' => $message['oid']]); + DBA::update('term', $fields, ['otype' => Term::OBJECT_TYPE_POST, 'oid' => $message['oid']]); } DBA::close($messages); From 67a67200a769f2e94c8d341453f32acccf217092 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 13 Apr 2020 23:54:28 +0000 Subject: [PATCH 02/43] Storing mentions in Diaspora and AP --- src/Protocol/ActivityPub/Processor.php | 50 ++++++++++++++++++++++++ src/Protocol/Diaspora.php | 53 ++++++++++++++++++++++++++ static/dbstructure.config.php | 15 +++++++- 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 7a8179b05e..cf1fd114c7 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -32,6 +32,7 @@ use Friendica\Model\Contact; use Friendica\Model\Conversation; use Friendica\Model\Event; use Friendica\Model\Item; +use Friendica\Model\ItemURI; use Friendica\Model\Mail; use Friendica\Model\Term; use Friendica\Model\User; @@ -403,6 +404,8 @@ class Processor $item['tag'] = self::constructTagString($activity['tags'], $activity['sensitive']); + self::storeTags($item['uri-id'], $activity['tags'], $activity['sensitive']); + $item['location'] = $activity['location']; if (!empty($item['latitude']) && !empty($item['longitude'])) { @@ -496,6 +499,8 @@ class Processor $item['edited'] = DateTimeFormat::utc($activity['updated']); $item['guid'] = $activity['diaspora:guid'] ?: $activity['sc:identifier'] ?: self::getGUIDByURL($item['uri']); + $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]); + $item = self::processContent($activity, $item); if (empty($item)) { return; @@ -571,6 +576,51 @@ class Processor } } + private static function storeTags(int $uriid, array $tags = null, $sensitive = false) + { + // Make sure to delete all existing tags (can happen when called via the update functionality) + DBA::delete('tag', ['uri-id' => $uriid]); + + foreach ($tags as $tag) { + if (empty($tag['name']) || empty($tag['type']) || !in_array($tag['type'], ['Mention', 'Hashtag'])) { + continue; + } + + $fields = ['uri-id' => $uriid, 'name' => $tag['name']]; + + if ($tag['type'] == 'Mention') { + $fields['type'] = Term::MENTION; + + if (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::MENTION]) { + $fields['name'] = substr($fields['name'], 1); + } elseif (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]) { + $fields['type'] = Term::EXCLUSIVE_MENTION; + $fields['name'] = substr($fields['name'], 1); + } elseif (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::IMPLICIT_MENTION]) { + $fields['type'] = Term::IMPLICIT_MENTION; + $fields['name'] = substr($fields['name'], 1); + } + } elseif ($tag['type'] == 'Hashtag') { + $fields['type'] = Term::HASHTAG; + if (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::HASHTAG]) { + $fields['name'] = substr($fields['name'], 1); + } + } + + if (empty($fields['name'])) { + continue; + } + + if (!empty($tag['href'] && ($tag['href'] != $tag['name']))) { + $fields['url'] = $tag['href']; + } + + DBA::insert('tag', $fields, true); + + Logger::info('Got Tag', ['uriid' => $uriid, 'tag' => $tag, 'sensitive' => $sensitive, 'fields' => $fields]); + } + } + /** * Creates an mail post * diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 8bb407ebd8..96c7c7fe18 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -35,6 +35,7 @@ use Friendica\Model\Contact; use Friendica\Model\Conversation; use Friendica\Model\GContact; use Friendica\Model\Item; +use Friendica\Model\ItemURI; use Friendica\Model\ItemDeliveryData; use Friendica\Model\Mail; use Friendica\Model\Profile; @@ -1808,6 +1809,47 @@ class Diaspora return false; } + private static function storeMentions(int $uriid, string $text) + { + preg_match_all('/([@!]){(?:([^}]+?); ?)?([^} ]+)}/', $text, $matches, PREG_SET_ORDER); + if (empty($matches)) { + return; + } + + /* + * Matching values for the preg match + * [1] = mention type (@ or !) + * [2] = name (optional) + * [3] = profile URL + */ + + foreach ($matches as $match) { + if (empty($match)) { + continue; + } + + $person = self::personByHandle($match[3]); + if (empty($person)) { + continue; + } + + $fields = ['uri-id' => $uriid, 'name' => $person['addr'], 'url' => $person['url']]; + + if ($match[1] == Term::TAG_CHARACTER[Term::MENTION]) { + $fields['type'] = Term::MENTION; + } elseif ($match[1] == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]) { + $fields['type'] = Term::EXCLUSIVE_MENTION; + } elseif ($match[1] == Term::TAG_CHARACTER[Term::IMPLICIT_MENTION]) { + $fields['type'] = Term::IMPLICIT_MENTION; + } else { + continue; + } + + DBA::insert('tag', $fields, true); + Logger::info('Stored mention', ['uriid' => $uriid, 'match' => $match, 'fields' => $fields]); + } + } + /** * Processes an incoming comment * @@ -1878,6 +1920,7 @@ class Diaspora $datarray["guid"] = $guid; $datarray["uri"] = self::getUriFromGuid($author, $guid); + $datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]); $datarray["verb"] = Activity::POST; $datarray["gravity"] = GRAVITY_COMMENT; @@ -1896,6 +1939,9 @@ class Diaspora $datarray["changed"] = $datarray["created"] = $datarray["edited"] = $created_at; $datarray["plink"] = self::plink($author, $guid, $parent_item['guid']); + + self::storeMentions($datarray['uri-id'], $text); + $body = Markdown::toBBCode($text); $datarray["body"] = self::replacePeopleGuid($body, $person["url"]); @@ -2642,6 +2688,7 @@ class Diaspora $datarray['guid'] = $parent['guid'] . '-' . $guid; $datarray['uri'] = self::getUriFromGuid($author, $datarray['guid']); + $datarray['parent-uri'] = $parent['uri']; $datarray['verb'] = $datarray['body'] = Activity::ANNOUNCE; @@ -2716,6 +2763,7 @@ class Diaspora $datarray["guid"] = $guid; $datarray["uri"] = $datarray["parent-uri"] = self::getUriFromGuid($author, $guid); + $datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]); $datarray["verb"] = Activity::POST; $datarray["gravity"] = GRAVITY_PARENT; @@ -2723,6 +2771,8 @@ class Diaspora $datarray["protocol"] = Conversation::PARCEL_DIASPORA; $datarray["source"] = $xml; + /// @todo Copy tag data from original post + $prefix = share_header( $original_item["author-name"], $original_item["author-link"], @@ -2959,6 +3009,7 @@ class Diaspora $datarray["guid"] = $guid; $datarray["uri"] = $datarray["parent-uri"] = self::getUriFromGuid($author, $guid); + $datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]); $datarray["verb"] = Activity::POST; $datarray["gravity"] = GRAVITY_PARENT; @@ -2966,6 +3017,8 @@ class Diaspora $datarray["protocol"] = Conversation::PARCEL_DIASPORA; $datarray["source"] = $xml; + self::storeMentions($datarray['uri-id'], $text); + $datarray["body"] = self::replacePeopleGuid($body, $contact["url"]); if ($provider_display_name != "") { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 94d5a36025..110842cbc0 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -51,7 +51,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1338); + define('DB_UPDATE_VERSION', 1339); } return [ @@ -1292,6 +1292,19 @@ return [ "guid" => ["guid(64)"], ] ], + "tag" => [ + "comment" => "item tags and mentions", + "fields" => [ + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""], + "name" => ["type" => "varchar(64)", "not null" => "1", "default" => "", "primary" => "1", "comment" => ""], + "url" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""] + ], + "indexes" => [ + "PRIMARY" => ["uri-id", "type", "name"], + "type_name" => ["type", "name"] + ] + ], "thread" => [ "comment" => "Thread related data", "fields" => [ From 6fa43ffa71d25c1125f9b5db0810e9f0e1e0ece0 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 14 Apr 2020 07:56:53 +0000 Subject: [PATCH 03/43] Store tags for Diaspora - shorten tags when needed --- src/Protocol/ActivityPub/Processor.php | 2 ++ src/Protocol/Diaspora.php | 33 +++++++++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index cf1fd114c7..b75c11e386 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -609,6 +609,8 @@ class Processor if (empty($fields['name'])) { continue; + } else { + $fields['name'] = substr($fields['name'], 0, 64); } if (!empty($tag['href'] && ($tag['href'] != $tag['name']))) { diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 96c7c7fe18..7c8fd81f7c 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1833,7 +1833,7 @@ class Diaspora continue; } - $fields = ['uri-id' => $uriid, 'name' => $person['addr'], 'url' => $person['url']]; + $fields = ['uri-id' => $uriid, 'name' => substr($person['addr'], 0, 64), 'url' => $person['url']]; if ($match[1] == Term::TAG_CHARACTER[Term::MENTION]) { $fields['type'] = Term::MENTION; @@ -1850,6 +1850,25 @@ class Diaspora } } + private static function storeTags(int $uriid, string $body) + { + $tags = BBCode::getTags($body); + if (empty($tags)) { + return; + } + + foreach ($tags as $tag) { + if ((substr($tag, 0, 1) != Term::TAG_CHARACTER[Term::HASHTAG]) || (strlen($tag) <= 1)) { + Logger::info('Skip tag', ['uriid' => $uriid, 'tag' => $tag]); + continue; + } + + $fields = ['uri-id' => $uriid, 'name' => substr($tag, 1, 64), 'type' => Term::HASHTAG]; + DBA::insert('tag', $fields, true); + Logger::info('Stored tag', ['uriid' => $uriid, 'tag' => $tag, 'fields' => $fields]); + } + } + /** * Processes an incoming comment * @@ -1939,13 +1958,14 @@ class Diaspora $datarray["changed"] = $datarray["created"] = $datarray["edited"] = $created_at; $datarray["plink"] = self::plink($author, $guid, $parent_item['guid']); - - self::storeMentions($datarray['uri-id'], $text); - + $body = Markdown::toBBCode($text); $datarray["body"] = self::replacePeopleGuid($body, $person["url"]); + self::storeMentions($datarray['uri-id'], $text); + self::storeTags($datarray['uri-id'], $datarray["body"]); + self::fetchGuid($datarray); // If we are the origin of the parent we store the original data. @@ -3017,10 +3037,11 @@ class Diaspora $datarray["protocol"] = Conversation::PARCEL_DIASPORA; $datarray["source"] = $xml; - self::storeMentions($datarray['uri-id'], $text); - $datarray["body"] = self::replacePeopleGuid($body, $contact["url"]); + self::storeMentions($datarray['uri-id'], $text); + self::storeTags($datarray['uri-id'], $datarray["body"]); + if ($provider_display_name != "") { $datarray["app"] = $provider_display_name; } From df898bd3d3e226aece26747f7fe5f06674aeb9d5 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 14 Apr 2020 09:19:45 +0000 Subject: [PATCH 04/43] Improved logging --- src/Protocol/ActivityPub/Processor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index b75c11e386..ef7fb6f4e9 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -619,7 +619,7 @@ class Processor DBA::insert('tag', $fields, true); - Logger::info('Got Tag', ['uriid' => $uriid, 'tag' => $tag, 'sensitive' => $sensitive, 'fields' => $fields]); + Logger::info('Stored tag/mention', ['uriid' => $uriid, 'tag' => $tag, 'sensitive' => $sensitive, 'fields' => $fields]); } } From 3f4c85dead20e3ea6bd5072f1375828e1110ae2c Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 14 Apr 2020 16:52:53 +0000 Subject: [PATCH 05/43] Store the tags with DFRN as well --- src/Protocol/DFRN.php | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 6fb63c3d46..8c7d42fbf4 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -35,6 +35,7 @@ use Friendica\Model\Conversation; use Friendica\Model\Event; use Friendica\Model\GContact; use Friendica\Model\Item; +use Friendica\Model\ItemURI; use Friendica\Model\Mail; use Friendica\Model\Notify\Type; use Friendica\Model\PermissionSet; @@ -2404,6 +2405,8 @@ class DFRN $item["guid"] = XML::getFirstNodeValue($xpath, "dfrn:diaspora_guid/text()", $entry); + $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]); + // We store the data from "dfrn:diaspora_signature" in a different table, this is done in "Item::insert" $dsprsig = XML::unescape(XML::getFirstNodeValue($xpath, "dfrn:diaspora_signature/text()", $entry)); if ($dsprsig != "") { @@ -2457,6 +2460,27 @@ class DFRN } $item["tag"] .= $termhash . "[url=" . $termurl . "]" . $term . "[/url]"; + + // Store the hashtag/mention + $fields = ['uri-id' => $item['uri-id'], 'name' => substr($term, 0, 64)]; + + if ($termhash == Term::TAG_CHARACTER[Term::MENTION]) { + $fields['type'] = Term::EXCLUSIVE_MENTION; + } elseif ($termhash == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]) { + $fields['type'] = Term::EXCLUSIVE_MENTION; + } elseif ($termhash == Term::TAG_CHARACTER[Term::IMPLICIT_MENTION]) { + $fields['type'] = Term::IMPLICIT_MENTION; + } elseif ($termhash == Term::TAG_CHARACTER[Term::HASHTAG]) { + $fields['type'] = Term::IMPLICIT_MENTION; + } + + if (!empty($termurl)) { + $fields['url'] = $termurl; + } + + DBA::insert('tag', $fields, true); + + Logger::info('Stored tag/mention', ['uri-id' => $item['uri-id'], 'tag' => $term, 'url' => $termurl, 'hash' => $termhash, 'fields' => $fields]); } } } From 1e702d41451ec7607a6d84885f4a3a9a5cba0c09 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 14 Apr 2020 17:00:56 +0000 Subject: [PATCH 06/43] Corrections for DFRN, added support for OStatus --- src/Protocol/DFRN.php | 4 ++-- src/Protocol/OStatus.php | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 8c7d42fbf4..a14513ab98 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -2465,13 +2465,13 @@ class DFRN $fields = ['uri-id' => $item['uri-id'], 'name' => substr($term, 0, 64)]; if ($termhash == Term::TAG_CHARACTER[Term::MENTION]) { - $fields['type'] = Term::EXCLUSIVE_MENTION; + $fields['type'] = Term::MENTION; } elseif ($termhash == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]) { $fields['type'] = Term::EXCLUSIVE_MENTION; } elseif ($termhash == Term::TAG_CHARACTER[Term::IMPLICIT_MENTION]) { $fields['type'] = Term::IMPLICIT_MENTION; } elseif ($termhash == Term::TAG_CHARACTER[Term::HASHTAG]) { - $fields['type'] = Term::IMPLICIT_MENTION; + $fields['type'] = Term::HASHTAG; } if (!empty($termurl)) { diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index bf3a6ef9d5..e155708aec 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -35,6 +35,8 @@ use Friendica\Model\Contact; use Friendica\Model\Conversation; use Friendica\Model\GContact; use Friendica\Model\Item; +use Friendica\Model\ItemURI; +use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Util\DateTimeFormat; @@ -437,6 +439,7 @@ class OStatus $item = array_merge($header, $author); $item["uri"] = XML::getFirstNodeValue($xpath, 'atom:id/text()', $entry); + $item['uri-id'] = ItemURI::insert(['uri' => $item['uri']]); $item["verb"] = XML::getFirstNodeValue($xpath, 'activity:verb/text()', $entry); @@ -660,6 +663,12 @@ class OStatus } $item['tag'] .= '#[url=' . DI::baseUrl() . '/search?tag=' . $term . ']' . $term . '[/url]'; + + // Store the hashtag + $fields = ['uri-id' => $item['uri-id'], 'name' => substr($term, 0, 64), 'type' => Term::HASHTAG]; + DBA::insert('tag', $fields, true); + + Logger::info('Stored tag', ['uri-id' => $item['uri-id'], 'tag' => $term, 'fields' => $fields]); } } } From 010491e0b009db7f203848848f3b9cda51c5d3c5 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 14 Apr 2020 17:18:48 +0000 Subject: [PATCH 07/43] Don't use "sensitive" --- src/Protocol/ActivityPub/Processor.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index ef7fb6f4e9..89cff65368 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -170,12 +170,16 @@ class Processor */ public static function updateItem($activity) { - $item = Item::selectFirst(['uri', 'thr-parent', 'gravity'], ['uri' => $activity['id']]); + $item = Item::selectFirst(['uri', 'uri-id', 'guid', 'thr-parent', 'gravity'], ['uri' => $activity['id']]); if (!DBA::isResult($item)) { Logger::warning('Unknown item', ['uri' => $activity['id']]); return; } + if (empty($item['uri-id'])) { + $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]); + } + $item['changed'] = DateTimeFormat::utcNow(); $item['edited'] = DateTimeFormat::utc($activity['updated']); @@ -404,7 +408,7 @@ class Processor $item['tag'] = self::constructTagString($activity['tags'], $activity['sensitive']); - self::storeTags($item['uri-id'], $activity['tags'], $activity['sensitive']); + self::storeTags($item['uri-id'], $activity['tags']); $item['location'] = $activity['location']; @@ -576,7 +580,7 @@ class Processor } } - private static function storeTags(int $uriid, array $tags = null, $sensitive = false) + private static function storeTags(int $uriid, array $tags = null) { // Make sure to delete all existing tags (can happen when called via the update functionality) DBA::delete('tag', ['uri-id' => $uriid]); @@ -619,7 +623,7 @@ class Processor DBA::insert('tag', $fields, true); - Logger::info('Stored tag/mention', ['uriid' => $uriid, 'tag' => $tag, 'sensitive' => $sensitive, 'fields' => $fields]); + Logger::info('Stored tag/mention', ['uriid' => $uriid, 'tag' => $tag, 'fields' => $fields]); } } From 9a0d2c9e0c9879a221559564ee99c5aaff929329 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 05:10:40 +0000 Subject: [PATCH 08/43] The new tag table should work for feeds no as well --- include/items.php | 10 ++++++++-- src/Protocol/ActivityPub/Processor.php | 6 +----- src/Protocol/Feed.php | 19 ++++++++++++++++++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/include/items.php b/include/items.php index 4c7551e5c4..6068be4b94 100644 --- a/include/items.php +++ b/include/items.php @@ -141,11 +141,12 @@ function query_page_info($url, $photo = "", $keywords = false, $keyword_blacklis return $data; } -function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "") +function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "", $return_array = false) { $data = query_page_info($url, $photo, $keywords, $keyword_blacklist); $tags = ""; + $taglist = []; if (isset($data["keywords"]) && count($data["keywords"])) { foreach ($data["keywords"] as $keyword) { $hashtag = str_replace([" ", "+", "/", ".", "#", "'"], @@ -156,10 +157,15 @@ function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blackl } $tags .= "#[url=" . DI::baseUrl() . "/search?tag=" . $hashtag . "]" . $hashtag . "[/url]"; + $taglist[] = $hashtag; } } - return $tags; + if ($return_array) { + return $taglist; + } else { + return $tags; + } } function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 89cff65368..a29414e941 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -170,16 +170,12 @@ class Processor */ public static function updateItem($activity) { - $item = Item::selectFirst(['uri', 'uri-id', 'guid', 'thr-parent', 'gravity'], ['uri' => $activity['id']]); + $item = Item::selectFirst(['uri', 'uri-id', 'thr-parent', 'gravity'], ['uri' => $activity['id']]); if (!DBA::isResult($item)) { Logger::warning('Unknown item', ['uri' => $activity['id']]); return; } - if (empty($item['uri-id'])) { - $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]); - } - $item['changed'] = DateTimeFormat::utcNow(); $item['edited'] = DateTimeFormat::utc($activity['updated']); diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 397edf3b41..0ce4144387 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -29,6 +29,7 @@ use Friendica\Core\Protocol; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; +use Friendica\Model\Term; use Friendica\Util\Network; use Friendica\Util\ParseUrl; use Friendica\Util\XML; @@ -385,6 +386,7 @@ class Feed { } $tags = ''; + $taglist = []; $categories = $xpath->query("category", $entry); foreach ($categories AS $category) { $hashtag = $category->nodeValue; @@ -394,6 +396,7 @@ class Feed { $taglink = "#[url=" . DI::baseUrl() . "/search?tag=" . $hashtag . "]" . $hashtag . "[/url]"; $tags .= $taglink; + $taglist[] = $hashtag; } $body = trim(XML::getFirstNodeValue($xpath, 'atom:content/text()', $entry)); @@ -475,6 +478,7 @@ class Feed { $item["title"] = ""; $item["body"] = $item["body"] . add_page_info($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]); $item["tag"] = add_page_keywords($item["plink"], $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]); + $taglist = add_page_keywords($item["plink"], $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"], true); $item["object-type"] = Activity\ObjectType::BOOKMARK; unset($item["attach"]); } else { @@ -488,8 +492,11 @@ class Feed { } else { // @todo $preview is never set in this case, is it intended? - @MrPetovan 2018-02-13 $item["tag"] = add_page_keywords($item["plink"], $preview, true, $contact["ffi_keyword_blacklist"]); + $taglist = add_page_keywords($item["plink"], $preview, true, $contact["ffi_keyword_blacklist"], true); } $item["body"] .= "\n" . $item['tag']; + } else { + $taglist = []; } // Add the link to the original feed entry if not present in feed @@ -516,10 +523,20 @@ class Feed { // Set the delivery priority for "remote self" to "medium" $notify = PRIORITY_MEDIUM; } - + $id = Item::insert($item, false, $notify); Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); + + if (!empty($id) && !empty($taglist)) { + $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); + foreach ($taglist as $tag) { + $fields = ['uri-id' => $feeditem['uri-id'], 'name' => substr($tag, 0, 64), 'type' => Term::HASHTAG]; + DBA::insert('tag', $fields, true); + + Logger::info('Stored tag', ['uri-id' => $feeditem['uri-id'], 'tag' => $tag, 'fields' => $fields]); + } + } } } From e5a0ff9572db67952fc2bd601b17a36e45896898 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 05:11:45 +0000 Subject: [PATCH 09/43] Fix for missing fields --- src/Model/Item.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index b9c01809fd..267ac89363 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -342,7 +342,7 @@ class Item } } - if (array_key_exists('signed_text', $row) && array_key_exists('interaction', $row) && !is_null($row['interaction'])) { + if (array_key_exists('interaction', $row)) { $row['signed_text'] = $row['interaction']; } @@ -672,7 +672,8 @@ class Item { $fields = []; - $fields['item'] = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid', + $fields['item'] = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', + 'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'contact-id', 'owner-id', 'author-id', 'type', 'wall', 'gravity', 'extid', 'created', 'edited', 'commented', 'received', 'changed', 'psid', 'resource-id', 'event-id', 'tag', 'attach', 'post-type', 'file', From cb593226bf95731ecd3a0e69e353ea4cbffb1ce4 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 11:39:00 +0000 Subject: [PATCH 10/43] Spaces and newlines --- src/Protocol/Diaspora.php | 2 -- src/Protocol/Feed.php | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index a6369a2bfa..5aa3463181 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1958,7 +1958,6 @@ class Diaspora $datarray["changed"] = $datarray["created"] = $datarray["edited"] = $created_at; $datarray["plink"] = self::plink($author, $guid, $parent_item['guid']); - $body = Markdown::toBBCode($text); $datarray["body"] = self::replacePeopleGuid($body, $person["url"]); @@ -2708,7 +2707,6 @@ class Diaspora $datarray['guid'] = $parent['guid'] . '-' . $guid; $datarray['uri'] = self::getUriFromGuid($author, $datarray['guid']); - $datarray['parent-uri'] = $parent['uri']; $datarray['verb'] = $datarray['body'] = Activity::ANNOUNCE; diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 0ce4144387..8171348409 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -523,7 +523,7 @@ class Feed { // Set the delivery priority for "remote self" to "medium" $notify = PRIORITY_MEDIUM; } - + $id = Item::insert($item, false, $notify); Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); From 2e0501e7c5a51964bb91eba467459fe4bab1b2a4 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 16:37:09 +0000 Subject: [PATCH 11/43] New model class for tag --- src/Model/Tag.php | 81 +++++++++++++++++++++++++++++++++++++++ src/Protocol/DFRN.php | 3 ++ src/Protocol/Diaspora.php | 22 +---------- 3 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 src/Model/Tag.php diff --git a/src/Model/Tag.php b/src/Model/Tag.php new file mode 100644 index 0000000000..2a09ffdd30 --- /dev/null +++ b/src/Model/Tag.php @@ -0,0 +1,81 @@ +. + * + */ + +namespace Friendica\Model; + +use Friendica\Core\Logger; +use Friendica\Database\DBA; +use Friendica\Content\Text\BBCode; + +/** + * Class Tag + * + * This Model class handles tag table interactions. + * This tables stores relevant tags related to posts, like hashtags and mentions. + */ +class Tag +{ + const UNKNOWN = 0; + const HASHTAG = 1; + const MENTION = 2; + const CATEGORY = 3; + const FILE = 5; + /** + * An implicit mention is a mention in a comment body that is redundant with the threading information. + */ + const IMPLICIT_MENTION = 8; + /** + * An exclusive mention transfers the ownership of the post to the target account, usually a forum. + */ + const EXCLUSIVE_MENTION = 9; + + const TAG_CHARACTER = [ + self::HASHTAG => '#', + self::MENTION => '@', + self::IMPLICIT_MENTION => '%', + self::EXCLUSIVE_MENTION => '!', + ]; + + /** + * Store tags from the body + * + * @param integer $uriid + * @param string $body + */ + public static function storeFromBody(int $uriid, string $body) + { + $tags = BBCode::getTags($body); + if (empty($tags)) { + return; + } + + foreach ($tags as $tag) { + if ((substr($tag, 0, 1) != Term::TAG_CHARACTER[Term::HASHTAG]) || (strlen($tag) <= 1)) { + Logger::info('Skip tag', ['uriid' => $uriid, 'tag' => $tag]); + continue; + } + + $fields = ['uri-id' => $uriid, 'name' => substr($tag, 1, 64), 'type' => Term::HASHTAG]; + DBA::insert('tag', $fields, true); + Logger::info('Stored tag', ['uriid' => $uriid, 'tag' => $tag, 'fields' => $fields]); + } + } +} diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 57edfb02b7..9ab8bc70c9 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -40,6 +40,7 @@ use Friendica\Model\Mail; use Friendica\Model\Notify\Type; use Friendica\Model\PermissionSet; use Friendica\Model\Profile; +use Friendica\Model\Tag; use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Network\Probe; @@ -2407,6 +2408,8 @@ class DFRN $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]); + Tag::storeFromBody($item['uri-id'], $item["body"]); + // We store the data from "dfrn:diaspora_signature" in a different table, this is done in "Item::insert" $dsprsig = XML::unescape(XML::getFirstNodeValue($xpath, "dfrn:diaspora_signature/text()", $entry)); if ($dsprsig != "") { diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 5aa3463181..ee2830362b 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -39,6 +39,7 @@ use Friendica\Model\ItemURI; use Friendica\Model\ItemDeliveryData; use Friendica\Model\Mail; use Friendica\Model\Profile; +use Friendica\Model\Tag; use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Network\Probe; @@ -1850,25 +1851,6 @@ class Diaspora } } - private static function storeTags(int $uriid, string $body) - { - $tags = BBCode::getTags($body); - if (empty($tags)) { - return; - } - - foreach ($tags as $tag) { - if ((substr($tag, 0, 1) != Term::TAG_CHARACTER[Term::HASHTAG]) || (strlen($tag) <= 1)) { - Logger::info('Skip tag', ['uriid' => $uriid, 'tag' => $tag]); - continue; - } - - $fields = ['uri-id' => $uriid, 'name' => substr($tag, 1, 64), 'type' => Term::HASHTAG]; - DBA::insert('tag', $fields, true); - Logger::info('Stored tag', ['uriid' => $uriid, 'tag' => $tag, 'fields' => $fields]); - } - } - /** * Processes an incoming comment * @@ -1963,7 +1945,7 @@ class Diaspora $datarray["body"] = self::replacePeopleGuid($body, $person["url"]); self::storeMentions($datarray['uri-id'], $text); - self::storeTags($datarray['uri-id'], $datarray["body"]); + Tag::storeFromBody($datarray['uri-id'], $datarray["body"]); self::fetchGuid($datarray); From 4808aa431a751ef66dc3c457caef48a2de7d2029 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 16:38:52 +0000 Subject: [PATCH 12/43] Use the own constants --- src/Model/Tag.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 2a09ffdd30..beda393b7e 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -68,12 +68,12 @@ class Tag } foreach ($tags as $tag) { - if ((substr($tag, 0, 1) != Term::TAG_CHARACTER[Term::HASHTAG]) || (strlen($tag) <= 1)) { + if ((substr($tag, 0, 1) != self::TAG_CHARACTER[self::HASHTAG]) || (strlen($tag) <= 1)) { Logger::info('Skip tag', ['uriid' => $uriid, 'tag' => $tag]); continue; } - $fields = ['uri-id' => $uriid, 'name' => substr($tag, 1, 64), 'type' => Term::HASHTAG]; + $fields = ['uri-id' => $uriid, 'name' => substr($tag, 1, 64), 'type' => self::HASHTAG]; DBA::insert('tag', $fields, true); Logger::info('Stored tag', ['uriid' => $uriid, 'tag' => $tag, 'fields' => $fields]); } From a4fd5033ddf1206f162b0d0a9adfb4bda72c3c0e Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 19:42:02 +0000 Subject: [PATCH 13/43] Function had been moved --- src/Protocol/Diaspora.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index ee2830362b..f7a63da79c 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -3020,7 +3020,7 @@ class Diaspora $datarray["body"] = self::replacePeopleGuid($body, $contact["url"]); self::storeMentions($datarray['uri-id'], $text); - self::storeTags($datarray['uri-id'], $datarray["body"]); + Tag::storeFromBody($datarray['uri-id'], $datarray["body"]); if ($provider_display_name != "") { $datarray["app"] = $provider_display_name; From 3ff607deeeeea5706ad6ff18176e3c106a7e1dc1 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 19:46:47 +0000 Subject: [PATCH 14/43] Store tags for locally created posts --- mod/item.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mod/item.php b/mod/item.php index bc60506460..5785954bce 100644 --- a/mod/item.php +++ b/mod/item.php @@ -46,6 +46,7 @@ use Friendica\Model\FileTag; use Friendica\Model\Item; use Friendica\Model\Notify\Type; use Friendica\Model\Photo; +use Friendica\Model\Tag; use Friendica\Model\Term; use Friendica\Network\HTTPException; use Friendica\Object\EMail\ItemCCEMail; @@ -750,6 +751,8 @@ function item_post(App $a) { throw new HTTPException\InternalServerErrorException(DI::l10n()->t('Item couldn\'t be fetched.')); } + Tag::storeFromBody($datarray['uri-id'], $datarray['body']); + // update filetags in pconfig FileTag::updatePconfig($uid, $categories_old, $categories_new, 'category'); From 472518f0b5117362bcf5c49dbaa36095820986a3 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 20:45:04 +0000 Subject: [PATCH 15/43] Use the name as name --- src/Protocol/ActivityPub/Processor.php | 6 ++++++ src/Protocol/Diaspora.php | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index a29414e941..5f413f57b6 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -600,6 +600,12 @@ class Processor $fields['type'] = Term::IMPLICIT_MENTION; $fields['name'] = substr($fields['name'], 1); } + if (!empty($tag['href'])) { + $apcontact = APContact::getByURL($tag['href']); + if (!empty($apcontact['name'])) { + $fields['name'] = $apcontact['name']; + } + } } elseif ($tag['type'] == 'Hashtag') { $fields['type'] = Term::HASHTAG; if (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::HASHTAG]) { diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index f7a63da79c..2923125546 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1834,7 +1834,7 @@ class Diaspora continue; } - $fields = ['uri-id' => $uriid, 'name' => substr($person['addr'], 0, 64), 'url' => $person['url']]; + $fields = ['uri-id' => $uriid, 'name' => substr($person['name'] ?: $person['nick'], 0, 64), 'url' => $person['url']]; if ($match[1] == Term::TAG_CHARACTER[Term::MENTION]) { $fields['type'] = Term::MENTION; From f871105ef984efc5e093aa0068e438fa532014c7 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 20:52:30 +0000 Subject: [PATCH 16/43] Use name or nick --- src/Protocol/ActivityPub/Processor.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 5f413f57b6..5b12d99160 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -602,8 +602,8 @@ class Processor } if (!empty($tag['href'])) { $apcontact = APContact::getByURL($tag['href']); - if (!empty($apcontact['name'])) { - $fields['name'] = $apcontact['name']; + if (!empty($apcontact['name']) || !empty($apcontact['nick'])) { + $fields['name'] = $apcontact['name'] ?: $apcontact['nick']; } } } elseif ($tag['type'] == 'Hashtag') { From 126b95d873fc81187abe76e6b04144a0069daced Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Apr 2020 20:59:45 +0000 Subject: [PATCH 17/43] Added documentation header --- src/Protocol/ActivityPub/Processor.php | 6 ++++++ src/Protocol/Diaspora.php | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 5b12d99160..023100dc72 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -576,6 +576,12 @@ class Processor } } + /** + * Store tags and mentions into the tag table + * + * @param integer $uriid + * @param array $tags + */ private static function storeTags(int $uriid, array $tags = null) { // Make sure to delete all existing tags (can happen when called via the update functionality) diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 2923125546..5580234c64 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1810,6 +1810,12 @@ class Diaspora return false; } + /** + * Store the mentions in the tag table + * + * @param integer $uriid + * @param string $text + */ private static function storeMentions(int $uriid, string $text) { preg_match_all('/([@!]){(?:([^}]+?); ?)?([^} ]+)}/', $text, $matches, PREG_SET_ORDER); From a0d4e330b945f5555d45386c4c02df16fa516678 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 16 Apr 2020 04:20:06 +0000 Subject: [PATCH 18/43] Spaces --- src/Model/Tag.php | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index beda393b7e..7578372a9b 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -33,26 +33,26 @@ use Friendica\Content\Text\BBCode; */ class Tag { - const UNKNOWN = 0; - const HASHTAG = 1; - const MENTION = 2; - const CATEGORY = 3; - const FILE = 5; + const UNKNOWN = 0; + const HASHTAG = 1; + const MENTION = 2; + const CATEGORY = 3; + const FILE = 5; /** * An implicit mention is a mention in a comment body that is redundant with the threading information. */ - const IMPLICIT_MENTION = 8; + const IMPLICIT_MENTION = 8; /** * An exclusive mention transfers the ownership of the post to the target account, usually a forum. */ - const EXCLUSIVE_MENTION = 9; + const EXCLUSIVE_MENTION = 9; - const TAG_CHARACTER = [ - self::HASHTAG => '#', - self::MENTION => '@', - self::IMPLICIT_MENTION => '%', - self::EXCLUSIVE_MENTION => '!', - ]; + const TAG_CHARACTER = [ + self::HASHTAG => '#', + self::MENTION => '@', + self::IMPLICIT_MENTION => '%', + self::EXCLUSIVE_MENTION => '!', + ]; /** * Store tags from the body From 5ef05b370336fa7559ba5314c4f61c11e59b7ae6 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 16 Apr 2020 04:20:59 +0000 Subject: [PATCH 19/43] Added uri-id to the constant --- src/Model/Item.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index d24340d764..1f2faa2698 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -94,7 +94,8 @@ class Item const CONTENT_FIELDLIST = ['language']; // All fields in the item table - const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid', + const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', + 'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'contact-id', 'type', 'wall', 'gravity', 'extid', 'icid', 'iaid', 'psid', 'created', 'edited', 'commented', 'received', 'changed', 'verb', 'postopts', 'plink', 'resource-id', 'event-id', 'tag', 'attach', 'inform', From d3f4e4d6298e7f7d483c0452787bc6f9abae50a5 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 16 Apr 2020 08:21:02 +0000 Subject: [PATCH 20/43] Don't insert empty terms --- src/Model/Term.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Model/Term.php b/src/Model/Term.php index 928c35f207..13639f7700 100644 --- a/src/Model/Term.php +++ b/src/Model/Term.php @@ -330,6 +330,10 @@ class Term continue; } + if (empty($term)) { + continue; + } + if ($item['uid'] == 0) { $global = true; DBA::update('term', ['global' => true], ['otype' => self::OBJECT_TYPE_POST, 'guid' => $item['guid']]); From db657b0149cc6a29eeb62bc8ccc96a06f505a7bf Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 17 Apr 2020 06:35:20 +0000 Subject: [PATCH 21/43] We now store the tags in two separate tables --- include/items.php | 45 ++++++++++++++-------- src/Model/Tag.php | 52 ++++++++++++++++++++++++-- src/Protocol/ActivityPub/Processor.php | 43 +++++++++------------ src/Protocol/DFRN.php | 21 +---------- src/Protocol/Diaspora.php | 15 +------- src/Protocol/Feed.php | 11 ++---- src/Protocol/OStatus.php | 7 +--- static/dbstructure.config.php | 25 +++++++++---- 8 files changed, 121 insertions(+), 98 deletions(-) diff --git a/include/items.php b/include/items.php index 6068be4b94..2d73308582 100644 --- a/include/items.php +++ b/include/items.php @@ -141,31 +141,44 @@ function query_page_info($url, $photo = "", $keywords = false, $keyword_blacklis return $data; } -function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "", $return_array = false) +function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "") { $data = query_page_info($url, $photo, $keywords, $keyword_blacklist); + if (empty($data['keywords']) || !is_array($data['keywords'])) { + return ''; + } $tags = ""; - $taglist = []; - if (isset($data["keywords"]) && count($data["keywords"])) { - foreach ($data["keywords"] as $keyword) { - $hashtag = str_replace([" ", "+", "/", ".", "#", "'"], - ["", "", "", "", "", ""], $keyword); + foreach ($data["keywords"] as $keyword) { + $hashtag = str_replace([" ", "+", "/", ".", "#", "'"], + ["", "", "", "", "", ""], $keyword); - if ($tags != "") { - $tags .= ", "; - } - - $tags .= "#[url=" . DI::baseUrl() . "/search?tag=" . $hashtag . "]" . $hashtag . "[/url]"; - $taglist[] = $hashtag; + if ($tags != "") { + $tags .= ", "; } + + $tags .= "#[url=" . DI::baseUrl() . "/search?tag=" . $hashtag . "]" . $hashtag . "[/url]"; } - if ($return_array) { - return $taglist; - } else { - return $tags; + return $tags; +} + +function get_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "") +{ + $data = query_page_info($url, $photo, $keywords, $keyword_blacklist); + if (empty($data['keywords']) || !is_array($data['keywords'])) { + return []; } + + $taglist = []; + foreach ($data['keywords'] as $keyword) { + $hashtag = str_replace([" ", "+", "/", ".", "#", "'"], + ["", "", "", "", "", ""], $keyword); + + $taglist[] = $hashtag; + } + + return $taglist; } function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 7578372a9b..43885e493a 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -54,6 +54,54 @@ class Tag self::EXCLUSIVE_MENTION => '!', ]; + public static function store(int $uriid, int $type, string $name, string $url = '') + { + $name = trim($name, "\x00..\x20\xFF#!@"); + if (empty($name)) { + return; + } + + $fields = ['name' => substr($name, 0, 64), 'type' => $type]; + + if (!empty($url) && ($url != $name)) { + $fields['url'] = strtolower($url); + } + + $tag = DBA::selectFirst('tag', ['id'], $fields); + if (!DBA::isResult($tag)) { + DBA::insert('tag', $fields, true); + $tagid = DBA::lastInsertId(); + } else { + $tagid = $tag['id']; + } + + if (empty($tagid)) { + Logger::error('No tag id created', $fields); + return; + } + + DBA::insert('post-tag', ['uri-id' => $uriid, 'tid' => $tagid], true); + + Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'tag' => $fields]); + } + + public static function storeByHash(int $uriid, string $hash, string $name, string $url = '') + { + if ($hash == self::TAG_CHARACTER[self::MENTION]) { + $type = self::MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]) { + $type = self::EXCLUSIVE_MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::IMPLICIT_MENTION]) { + $type = self::IMPLICIT_MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::HASHTAG]) { + $type = self::HASHTAG; + } else { + return; + } + + self::store($uriid, $type, $name, $url); + } + /** * Store tags from the body * @@ -73,9 +121,7 @@ class Tag continue; } - $fields = ['uri-id' => $uriid, 'name' => substr($tag, 1, 64), 'type' => self::HASHTAG]; - DBA::insert('tag', $fields, true); - Logger::info('Stored tag', ['uriid' => $uriid, 'tag' => $tag, 'fields' => $fields]); + self::storeByHash($uriid, '#', $tag); } } } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 023100dc72..1b605c30ab 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -34,6 +34,7 @@ use Friendica\Model\Event; use Friendica\Model\Item; use Friendica\Model\ItemURI; use Friendica\Model\Mail; +use Friendica\Model\Tag; use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Protocol\Activity; @@ -585,53 +586,43 @@ class Processor private static function storeTags(int $uriid, array $tags = null) { // Make sure to delete all existing tags (can happen when called via the update functionality) - DBA::delete('tag', ['uri-id' => $uriid]); + DBA::delete('post-tag', ['uri-id' => $uriid]); foreach ($tags as $tag) { if (empty($tag['name']) || empty($tag['type']) || !in_array($tag['type'], ['Mention', 'Hashtag'])) { continue; } - $fields = ['uri-id' => $uriid, 'name' => $tag['name']]; + $hash = substr($tag['name'], 0, 1); if ($tag['type'] == 'Mention') { - $fields['type'] = Term::MENTION; - - if (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::MENTION]) { - $fields['name'] = substr($fields['name'], 1); - } elseif (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]) { - $fields['type'] = Term::EXCLUSIVE_MENTION; - $fields['name'] = substr($fields['name'], 1); - } elseif (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::IMPLICIT_MENTION]) { - $fields['type'] = Term::IMPLICIT_MENTION; - $fields['name'] = substr($fields['name'], 1); + if (in_array($hash, [Tag::TAG_CHARACTER[Tag::MENTION], + Tag::TAG_CHARACTER[Tag::EXCLUSIVE_MENTION], + Tag::TAG_CHARACTER[Tag::IMPLICIT_MENTION]])) { + $tag['name'] = substr($tag['name'], 1); + } else { + $hash = '#'; } + if (!empty($tag['href'])) { $apcontact = APContact::getByURL($tag['href']); if (!empty($apcontact['name']) || !empty($apcontact['nick'])) { - $fields['name'] = $apcontact['name'] ?: $apcontact['nick']; + $tag['name'] = $apcontact['name'] ?: $apcontact['nick']; } } } elseif ($tag['type'] == 'Hashtag') { - $fields['type'] = Term::HASHTAG; - if (substr($fields['name'], 0, 1) == Term::TAG_CHARACTER[Term::HASHTAG]) { - $fields['name'] = substr($fields['name'], 1); + if (substr($tag['name'], 0, 1) == Term::TAG_CHARACTER[Term::HASHTAG]) { + $tag['name'] = substr($tag['name'], 1); + } else { + $hash = '@'; } } - if (empty($fields['name'])) { + if (empty($tag['name'])) { continue; - } else { - $fields['name'] = substr($fields['name'], 0, 64); } - if (!empty($tag['href'] && ($tag['href'] != $tag['name']))) { - $fields['url'] = $tag['href']; - } - - DBA::insert('tag', $fields, true); - - Logger::info('Stored tag/mention', ['uriid' => $uriid, 'tag' => $tag, 'fields' => $fields]); + Tag::storeByHash($uriid, $hash, $tag['name'], $tag['href']); } } diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 9ab8bc70c9..32067613f1 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -2464,26 +2464,7 @@ class DFRN $item["tag"] .= $termhash . "[url=" . $termurl . "]" . $term . "[/url]"; - // Store the hashtag/mention - $fields = ['uri-id' => $item['uri-id'], 'name' => substr($term, 0, 64)]; - - if ($termhash == Term::TAG_CHARACTER[Term::MENTION]) { - $fields['type'] = Term::MENTION; - } elseif ($termhash == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]) { - $fields['type'] = Term::EXCLUSIVE_MENTION; - } elseif ($termhash == Term::TAG_CHARACTER[Term::IMPLICIT_MENTION]) { - $fields['type'] = Term::IMPLICIT_MENTION; - } elseif ($termhash == Term::TAG_CHARACTER[Term::HASHTAG]) { - $fields['type'] = Term::HASHTAG; - } - - if (!empty($termurl)) { - $fields['url'] = $termurl; - } - - DBA::insert('tag', $fields, true); - - Logger::info('Stored tag/mention', ['uri-id' => $item['uri-id'], 'tag' => $term, 'url' => $termurl, 'hash' => $termhash, 'fields' => $fields]); + Tag::storeByHash($item['uri-id'], $termhash, $term, $termurl); } } } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 5580234c64..20a2be4cc1 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1840,20 +1840,7 @@ class Diaspora continue; } - $fields = ['uri-id' => $uriid, 'name' => substr($person['name'] ?: $person['nick'], 0, 64), 'url' => $person['url']]; - - if ($match[1] == Term::TAG_CHARACTER[Term::MENTION]) { - $fields['type'] = Term::MENTION; - } elseif ($match[1] == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]) { - $fields['type'] = Term::EXCLUSIVE_MENTION; - } elseif ($match[1] == Term::TAG_CHARACTER[Term::IMPLICIT_MENTION]) { - $fields['type'] = Term::IMPLICIT_MENTION; - } else { - continue; - } - - DBA::insert('tag', $fields, true); - Logger::info('Stored mention', ['uriid' => $uriid, 'match' => $match, 'fields' => $fields]); + Tag::storeByHash($uriid, $match[1], $person['name'] ?: $person['nick'], $person['url']); } } diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 8171348409..c03f959865 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -29,7 +29,7 @@ use Friendica\Core\Protocol; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; -use Friendica\Model\Term; +use Friendica\Model\Tag; use Friendica\Util\Network; use Friendica\Util\ParseUrl; use Friendica\Util\XML; @@ -478,7 +478,7 @@ class Feed { $item["title"] = ""; $item["body"] = $item["body"] . add_page_info($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]); $item["tag"] = add_page_keywords($item["plink"], $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]); - $taglist = add_page_keywords($item["plink"], $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"], true); + $taglist = get_page_keywords($item["plink"], $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]); $item["object-type"] = Activity\ObjectType::BOOKMARK; unset($item["attach"]); } else { @@ -492,7 +492,7 @@ class Feed { } else { // @todo $preview is never set in this case, is it intended? - @MrPetovan 2018-02-13 $item["tag"] = add_page_keywords($item["plink"], $preview, true, $contact["ffi_keyword_blacklist"]); - $taglist = add_page_keywords($item["plink"], $preview, true, $contact["ffi_keyword_blacklist"], true); + $taglist = get_page_keywords($item["plink"], $preview, true, $contact["ffi_keyword_blacklist"]); } $item["body"] .= "\n" . $item['tag']; } else { @@ -531,10 +531,7 @@ class Feed { if (!empty($id) && !empty($taglist)) { $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); foreach ($taglist as $tag) { - $fields = ['uri-id' => $feeditem['uri-id'], 'name' => substr($tag, 0, 64), 'type' => Term::HASHTAG]; - DBA::insert('tag', $fields, true); - - Logger::info('Stored tag', ['uri-id' => $feeditem['uri-id'], 'tag' => $tag, 'fields' => $fields]); + Tag::storeByHash($feeditem['uri-id'], '#', $tag); } } } diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index e155708aec..7dad68550b 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -36,7 +36,7 @@ use Friendica\Model\Conversation; use Friendica\Model\GContact; use Friendica\Model\Item; use Friendica\Model\ItemURI; -use Friendica\Model\Term; +use Friendica\Model\Tag; use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Util\DateTimeFormat; @@ -665,10 +665,7 @@ class OStatus $item['tag'] .= '#[url=' . DI::baseUrl() . '/search?tag=' . $term . ']' . $term . '[/url]'; // Store the hashtag - $fields = ['uri-id' => $item['uri-id'], 'name' => substr($term, 0, 64), 'type' => Term::HASHTAG]; - DBA::insert('tag', $fields, true); - - Logger::info('Stored tag', ['uri-id' => $item['uri-id'], 'tag' => $term, 'fields' => $fields]); + Tag::storeByHash($item['uri-id'], '#', $term); } } } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 110842cbc0..204fca9bfe 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1293,16 +1293,27 @@ return [ ] ], "tag" => [ - "comment" => "item tags and mentions", + "comment" => "tags and mentions", "fields" => [ - "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], - "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""], - "name" => ["type" => "varchar(64)", "not null" => "1", "default" => "", "primary" => "1", "comment" => ""], - "url" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""] + "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => ""], + "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], + "name" => ["type" => "varchar(64)", "not null" => "1", "default" => "", "comment" => ""], + "url" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => ""] ], "indexes" => [ - "PRIMARY" => ["uri-id", "type", "name"], - "type_name" => ["type", "name"] + "PRIMARY" => ["id"], + "type_name_url" => ["UNIQUE", "type", "name", "url"] + ] + ], + "post-tag" => [ + "comment" => "post relation to tags", + "fields" => [ + "tid" => ["type" => "int unsigned", "not null" => "1", "relation" => ["tag" => "id"], "primary" => "1", "comment" => ""], + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + ], + "indexes" => [ + "PRIMARY" => ["tid", "uri-id"], + "uri-id" => ["uri-id"] ] ], "thread" => [ From 6afcf5c8c710aead40528889e7164c9986eaba62 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 17 Apr 2020 07:55:23 +0000 Subject: [PATCH 22/43] Use "store" when possible --- src/Protocol/Feed.php | 2 +- src/Protocol/OStatus.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index c03f959865..14a3c28ab9 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -531,7 +531,7 @@ class Feed { if (!empty($id) && !empty($taglist)) { $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); foreach ($taglist as $tag) { - Tag::storeByHash($feeditem['uri-id'], '#', $tag); + Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); } } } diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index 7dad68550b..d95810e3a6 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -665,7 +665,7 @@ class OStatus $item['tag'] .= '#[url=' . DI::baseUrl() . '/search?tag=' . $term . ']' . $term . '[/url]'; // Store the hashtag - Tag::storeByHash($item['uri-id'], '#', $term); + Tag::store($item['uri-id'], Tag::HASHTAG, $term); } } } From c446712d0880c27bdd797fe234ed5a70b3636a0b Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 17 Apr 2020 07:58:54 +0000 Subject: [PATCH 23/43] Added documentation --- src/Model/Tag.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 43885e493a..9c9ac033f0 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -54,6 +54,14 @@ class Tag self::EXCLUSIVE_MENTION => '!', ]; + /** + * Store tag/mention elements + * + * @param integer $uriid + * @param integer $type + * @param string $name + * @param string $url + */ public static function store(int $uriid, int $type, string $name, string $url = '') { $name = trim($name, "\x00..\x20\xFF#!@"); @@ -85,6 +93,14 @@ class Tag Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'tag' => $fields]); } + /** + * Store tag/mention elements + * + * @param integer $uriid + * @param string $hash + * @param string $name + * @param string $url + */ public static function storeByHash(int $uriid, string $hash, string $name, string $url = '') { if ($hash == self::TAG_CHARACTER[self::MENTION]) { From d9352f5a63ef013e66d969c908bd370199bdf801 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 17 Apr 2020 13:34:29 +0000 Subject: [PATCH 24/43] Use the new function when adding additional tags --- src/Protocol/ActivityPub/Processor.php | 4 +++- src/Protocol/DFRN.php | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 1b605c30ab..b770e8a0de 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -251,7 +251,7 @@ class Processor } foreach ($activity['receiver'] as $receiver) { - $item = Item::selectFirst(['id', 'tag', 'origin', 'author-link'], ['uri' => $activity['target_id'], 'uid' => $receiver]); + $item = Item::selectFirst(['id', 'uri-id', 'tag', 'origin', 'author-link'], ['uri' => $activity['target_id'], 'uid' => $receiver]); if (!DBA::isResult($item)) { // We don't fetch missing content for this purpose continue; @@ -262,6 +262,8 @@ class Processor continue; } + Tag::store($item['uri-id'], Tag::HASHTAG, $activity['object_content'], $activity['object_id']); + // To-Do: // - Check if "blocktag" is set // - Check if actor is a contact diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 32067613f1..8033ffb701 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -2243,7 +2243,7 @@ class DFRN $xt = XML::parseString($item["target"], false); if ($xt->type == Activity\ObjectType::NOTE) { - $item_tag = Item::selectFirst(['id', 'tag'], ['uri' => $xt->id, 'uid' => $importer["importer_uid"]]); + $item_tag = Item::selectFirst(['id', 'uri-id', 'tag'], ['uri' => $xt->id, 'uid' => $importer["importer_uid"]]); if (!DBA::isResult($item_tag)) { Logger::log("Query failed to execute, no result returned in " . __FUNCTION__); @@ -2252,6 +2252,8 @@ class DFRN // extract tag, if not duplicate, add to parent item if ($xo->content) { + Tag::store($item_tag['uri-id'], Tag::HASHTAG, $xo->content); + if (!stristr($item_tag["tag"], trim($xo->content))) { $tag = $item_tag["tag"] . (strlen($item_tag["tag"]) ? ',' : '') . '#[url=' . $xo->id . ']'. $xo->content . '[/url]'; Item::update(['tag' => $tag], ['id' => $item_tag["id"]]); From e365103ea38540e1f362941b6da3a679a5c4518b Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 17 Apr 2020 13:35:12 +0000 Subject: [PATCH 25/43] Added todo --- include/api.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/api.php b/include/api.php index 75e66349eb..36b08f08f8 100644 --- a/include/api.php +++ b/include/api.php @@ -2041,7 +2041,7 @@ function api_statuses_repeat($type) Logger::log('API: api_statuses_repeat: '.$id); - $fields = ['body', 'title', 'attach', 'tag', 'author-name', 'author-link', 'author-avatar', 'guid', 'created', 'plink']; + $fields = ['uri-id', 'body', 'title', 'attach', 'tag', 'author-name', 'author-link', 'author-avatar', 'guid', 'created', 'plink']; $item = Item::selectFirst($fields, ['id' => $id, 'private' => [Item::PUBLIC, Item::UNLISTED]]); if (DBA::isResult($item) && $item['body'] != "") { @@ -2069,6 +2069,8 @@ function api_statuses_repeat($type) } $item_id = item_post($a); + + /// @todo Copy tags from the original post to the new one } else { throw new ForbiddenException(); } From 060503eb44ff5664302e40f79f2348767e278997 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 17 Apr 2020 13:36:55 +0000 Subject: [PATCH 26/43] Tagger is now storing the tag as well --- mod/tagger.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mod/tagger.php b/mod/tagger.php index e3ae6b9b5c..b3ba472eab 100644 --- a/mod/tagger.php +++ b/mod/tagger.php @@ -28,6 +28,7 @@ use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; +use Friendica\Model\Tag; use Friendica\Model\Term; use Friendica\Protocol\Activity; use Friendica\Util\Strings; @@ -171,6 +172,8 @@ EOT; $term_objtype = ($item['resource-id'] ? Term::OBJECT_TYPE_PHOTO : Term::OBJECT_TYPE_POST); + Tag::store($item['uri-id'], Tag::HASHTAG, $term); + $t = q("SELECT count(tid) as tcount FROM term WHERE oid=%d AND term='%s'", intval($item['id']), DBA::escape($term) From 3ce9386cb1c80476e021467edd76803d004c1d91 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 18 Apr 2020 10:05:30 +0000 Subject: [PATCH 27/43] Improved body scanning, increased length --- src/Model/Tag.php | 25 ++++++++++--------------- src/Protocol/DFRN.php | 2 +- src/Protocol/Diaspora.php | 4 ++-- static/dbstructure.config.php | 2 +- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 9c9ac033f0..61a1575a1c 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -69,7 +69,7 @@ class Tag return; } - $fields = ['name' => substr($name, 0, 64), 'type' => $type]; + $fields = ['name' => substr($name, 0, 96), 'type' => $type]; if (!empty($url) && ($url != $name)) { $fields['url'] = strtolower($url); @@ -119,25 +119,20 @@ class Tag } /** - * Store tags from the body - * - * @param integer $uriid - * @param string $body + * Store tags and mentions from the body + * + * @param integer $uriid URI-Id + * @param string $body Body of the post + * @param string $tags Accepted tags */ - public static function storeFromBody(int $uriid, string $body) + public static function storeFromBody(int $uriid, string $body, string $tags = '#@!') { - $tags = BBCode::getTags($body); - if (empty($tags)) { + if (!preg_match_all("/([" . $tags . "])\[url\=(.*?)\](.*?)\[\/url\]/ism", $body, $result, PREG_SET_ORDER)) { return; } - foreach ($tags as $tag) { - if ((substr($tag, 0, 1) != self::TAG_CHARACTER[self::HASHTAG]) || (strlen($tag) <= 1)) { - Logger::info('Skip tag', ['uriid' => $uriid, 'tag' => $tag]); - continue; - } - - self::storeByHash($uriid, '#', $tag); + foreach ($result as $tag) { + self::storeByHash($uriid, $tag[1], $tag[3], $tag[2]); } } } diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 8033ffb701..653e5c242b 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -2410,7 +2410,7 @@ class DFRN $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]); - Tag::storeFromBody($item['uri-id'], $item["body"]); + Tag::storeFromBody($item['uri-id'], $item["body"], '#'); // We store the data from "dfrn:diaspora_signature" in a different table, this is done in "Item::insert" $dsprsig = XML::unescape(XML::getFirstNodeValue($xpath, "dfrn:diaspora_signature/text()", $entry)); diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 20a2be4cc1..a8ffcbcfe5 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1938,7 +1938,7 @@ class Diaspora $datarray["body"] = self::replacePeopleGuid($body, $person["url"]); self::storeMentions($datarray['uri-id'], $text); - Tag::storeFromBody($datarray['uri-id'], $datarray["body"]); + Tag::storeFromBody($datarray['uri-id'], $datarray["body"], '#'); self::fetchGuid($datarray); @@ -3013,7 +3013,7 @@ class Diaspora $datarray["body"] = self::replacePeopleGuid($body, $contact["url"]); self::storeMentions($datarray['uri-id'], $text); - Tag::storeFromBody($datarray['uri-id'], $datarray["body"]); + Tag::storeFromBody($datarray['uri-id'], $datarray["body"], '#'); if ($provider_display_name != "") { $datarray["app"] = $provider_display_name; diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 204fca9bfe..a1d72322a8 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1297,7 +1297,7 @@ return [ "fields" => [ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => ""], "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], - "name" => ["type" => "varchar(64)", "not null" => "1", "default" => "", "comment" => ""], + "name" => ["type" => "varchar(96)", "not null" => "1", "default" => "", "comment" => ""], "url" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => ""] ], "indexes" => [ From 539a5c5da1c46b96a743fd882e4c7c5263af6237 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 18 Apr 2020 10:38:08 +0000 Subject: [PATCH 28/43] Fixes RegExp --- src/Model/Tag.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 61a1575a1c..f0bddb25c0 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -127,7 +127,7 @@ class Tag */ public static function storeFromBody(int $uriid, string $body, string $tags = '#@!') { - if (!preg_match_all("/([" . $tags . "])\[url\=(.*?)\](.*?)\[\/url\]/ism", $body, $result, PREG_SET_ORDER)) { + if (!preg_match_all("/([" . $tags . "])\[url\=([^\[\]]*)\](.*?)\[\/url\]/ism", $body, $result, PREG_SET_ORDER)) { return; } From 5d34a90d6761d4c03253c34788287f96495147de Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 18 Apr 2020 14:41:26 +0000 Subject: [PATCH 29/43] Store mentioned contacts in another way --- src/Model/Tag.php | 53 +++++++++++++++++++++++------------ static/dbstructure.config.php | 14 +++++---- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index f0bddb25c0..7627872590 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -23,7 +23,7 @@ namespace Friendica\Model; use Friendica\Core\Logger; use Friendica\Database\DBA; -use Friendica\Content\Text\BBCode; +use Friendica\Model\Contact; /** * Class Tag @@ -69,28 +69,45 @@ class Tag return; } - $fields = ['name' => substr($name, 0, 96), 'type' => $type]; + if (in_array($type, [Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION])) { + if (empty($url)) { + // No mention without a contact url + return; + } - if (!empty($url) && ($url != $name)) { - $fields['url'] = strtolower($url); - } + Logger::info('Get ID for contact', ['url' => $url]); - $tag = DBA::selectFirst('tag', ['id'], $fields); - if (!DBA::isResult($tag)) { - DBA::insert('tag', $fields, true); - $tagid = DBA::lastInsertId(); + $cid = Contact::getIdForURL($url, 0, true); + if (empty($cid)) { + Logger::error('No contact found', ['url' => $url]); + return; + } + $tagid = 0; } else { - $tagid = $tag['id']; + $fields = ['name' => substr($name, 0, 96)]; + + if (!empty($url) && ($url != $name)) { + $fields['url'] = strtolower($url); + } + + $tag = DBA::selectFirst('tag', ['id'], $fields); + if (!DBA::isResult($tag)) { + DBA::insert('tag', $fields, true); + $tagid = DBA::lastInsertId(); + } else { + $tagid = $tag['id']; + } + + if (empty($tagid)) { + Logger::error('No tag id created', $fields); + return; + } + $cid = 0; } - if (empty($tagid)) { - Logger::error('No tag id created', $fields); - return; - } + DBA::insert('post-tag', ['uri-id' => $uriid, 'type' => $type, 'tid' => $tagid, 'cid' => $cid], true); - DBA::insert('post-tag', ['uri-id' => $uriid, 'tid' => $tagid], true); - - Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'tag' => $fields]); + Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid]); } /** @@ -127,7 +144,7 @@ class Tag */ public static function storeFromBody(int $uriid, string $body, string $tags = '#@!') { - if (!preg_match_all("/([" . $tags . "])\[url\=([^\[\]]*)\](.*?)\[\/url\]/ism", $body, $result, PREG_SET_ORDER)) { + if (!preg_match_all("/([" . $tags . "])\[url\=([^\[\]]*)\]([^\[\]]*)\[\/url\]/ism", $body, $result, PREG_SET_ORDER)) { return; } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index a1d72322a8..e0c3c5b70e 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1296,24 +1296,26 @@ return [ "comment" => "tags and mentions", "fields" => [ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => ""], - "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], "name" => ["type" => "varchar(96)", "not null" => "1", "default" => "", "comment" => ""], "url" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => ""] ], "indexes" => [ - "PRIMARY" => ["id"], - "type_name_url" => ["UNIQUE", "type", "name", "url"] + "PRIMARY" => ["id"], + "type_name_url" => ["UNIQUE", "name", "url"] ] ], "post-tag" => [ "comment" => "post relation to tags", "fields" => [ - "tid" => ["type" => "int unsigned", "not null" => "1", "relation" => ["tag" => "id"], "primary" => "1", "comment" => ""], "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""], + "tid" => ["type" => "int unsigned", "not null" => "1", "relation" => ["tag" => "id"], "primary" => "1", "comment" => ""], + "cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["contact" => "id"], "comment" => "Contact id of the mentioned public contact"], ], "indexes" => [ - "PRIMARY" => ["tid", "uri-id"], - "uri-id" => ["uri-id"] + "PRIMARY" => ["uri-id", "type", "tid", "cid"], + "uri-id" => ["tid"], + "cid" => ["tid"] ] ], "thread" => [ From 0f2215bc175f0d3832eb44cd3643edc16780923b Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 18 Apr 2020 16:00:06 +0000 Subject: [PATCH 30/43] Hybrid mode for non resolvable contacts --- src/Model/Tag.php | 23 +++++++++++++++-------- static/dbstructure.config.php | 3 ++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 7627872590..6de872a9e5 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -23,7 +23,7 @@ namespace Friendica\Model; use Friendica\Core\Logger; use Friendica\Database\DBA; -use Friendica\Model\Contact; +use Friendica\Util\Strings; /** * Class Tag @@ -69,6 +69,9 @@ class Tag return; } + $cid = 0; + $tagid = 0; + if (in_array($type, [Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION])) { if (empty($url)) { // No mention without a contact url @@ -77,13 +80,18 @@ class Tag Logger::info('Get ID for contact', ['url' => $url]); - $cid = Contact::getIdForURL($url, 0, true); - if (empty($cid)) { - Logger::error('No contact found', ['url' => $url]); - return; + $condition = ['nurl' => Strings::normaliseLink($url), 'uid' => 0, 'deleted' => false]; + $contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]); + if (DBA::isResult($contact)) { + $cid = $contact['id']; + } else { + // The contact wasn't found in the system (most likely some dead account) + // We ensure that we only store a single entry by overwriting the previous name + DBA::update('tag', ['name' => substr($name, 0, 96)], ['url' => $url]); } - $tagid = 0; - } else { + } + + if (empty($cid)) { $fields = ['name' => substr($name, 0, 96)]; if (!empty($url) && ($url != $name)) { @@ -102,7 +110,6 @@ class Tag Logger::error('No tag id created', $fields); return; } - $cid = 0; } DBA::insert('post-tag', ['uri-id' => $uriid, 'type' => $type, 'tid' => $tagid, 'cid' => $cid], true); diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index e0c3c5b70e..d6bd418f8a 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1301,7 +1301,8 @@ return [ ], "indexes" => [ "PRIMARY" => ["id"], - "type_name_url" => ["UNIQUE", "name", "url"] + "type_name_url" => ["UNIQUE", "name", "url"], + "url" => ["url"] ] ], "post-tag" => [ From c2d1d11123db338be75767b29dcc50929261e67d Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 18 Apr 2020 16:14:38 +0000 Subject: [PATCH 31/43] Use constants --- src/Model/Tag.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 6de872a9e5..c3d1c50162 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -149,8 +149,12 @@ class Tag * @param string $body Body of the post * @param string $tags Accepted tags */ - public static function storeFromBody(int $uriid, string $body, string $tags = '#@!') + public static function storeFromBody(int $uriid, string $body, string $tags = null) { + if (is_null($tags)) { + $tags = self::TAG_CHARACTER[self::HASHTAG] . self::TAG_CHARACTER[self::MENTION] . self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]; + } + if (!preg_match_all("/([" . $tags . "])\[url\=([^\[\]]*)\]([^\[\]]*)\[\/url\]/ism", $body, $result, PREG_SET_ORDER)) { return; } From 9e9a104320b4f6a20743bd5bee3f61329a44d2ca Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 18 Apr 2020 20:46:41 +0000 Subject: [PATCH 32/43] Tags can now be added and removed from photos --- mod/photos.php | 10 +++++++++- mod/tagrm.php | 9 ++++++++- src/Model/Tag.php | 46 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/mod/photos.php b/mod/photos.php index 3f558429d4..0107a179d0 100644 --- a/mod/photos.php +++ b/mod/photos.php @@ -36,6 +36,7 @@ use Friendica\Model\Contact; use Friendica\Model\Item; use Friendica\Model\Photo; use Friendica\Model\Profile; +use Friendica\Model\Tag; use Friendica\Model\User; use Friendica\Module\BaseProfile; use Friendica\Network\Probe; @@ -421,7 +422,7 @@ function photos_post(App $a) } if ($item_id) { - $item = Item::selectFirst(['tag', 'inform'], ['id' => $item_id, 'uid' => $page_owner_uid]); + $item = Item::selectFirst(['tag', 'inform', 'uri-id'], ['id' => $item_id, 'uid' => $page_owner_uid]); if (DBA::isResult($item)) { $old_tag = $item['tag']; @@ -521,10 +522,17 @@ function photos_post(App $a) $profile = str_replace(',', '%2c', $profile); $str_tags .= '@[url=' . $profile . ']' . $newname . '[/url]'; + + if (!empty($item['uri-id'])) { + Tag::store($item['uri-id'], Tag::MENTION, $newname, $profile); + } } } elseif (strpos($tag, '#') === 0) { $tagname = substr($tag, 1); $str_tags .= '#[url=' . DI::baseUrl() . "/search?tag=" . $tagname . ']' . $tagname . '[/url],'; + if (!empty($item['uri-id'])) { + Tag::store($item['uri-id'], Tag::HASHTAG, $tagname); + } } } } diff --git a/mod/tagrm.php b/mod/tagrm.php index 2fa75133ca..51000c9854 100644 --- a/mod/tagrm.php +++ b/mod/tagrm.php @@ -24,6 +24,7 @@ use Friendica\Content\Text\BBCode; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; +use Friendica\Model\Tag; use Friendica\Model\Term; use Friendica\Util\Strings; @@ -62,7 +63,7 @@ function update_tags($item_id, $tags){ return; } - $item = Item::selectFirst(['tag'], ['id' => $item_id, 'uid' => local_user()]); + $item = Item::selectFirst(['tag', 'uri-id'], ['id' => $item_id, 'uid' => local_user()]); if (!DBA::isResult($item)) { return; } @@ -70,6 +71,12 @@ function update_tags($item_id, $tags){ $old_tags = explode(',', $item['tag']); foreach ($tags as $new_tag) { + if (preg_match_all('/([#@!])\[url\=([^\[\]]*)\]([^\[\]]*)\[\/url\]/ism', $new_tag, $results, PREG_SET_ORDER)) { + foreach ($results as $tag) { + Tag::removeByHash($item['uri-id'], $tag[1], $tag[3], $tag[2]); + } + } + foreach ($old_tags as $index => $old_tag) { if (strcmp($old_tag, $new_tag) == 0) { unset($old_tags[$index]); diff --git a/src/Model/Tag.php b/src/Model/Tag.php index c3d1c50162..b9e04ddd7a 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -92,7 +92,7 @@ class Tag } if (empty($cid)) { - $fields = ['name' => substr($name, 0, 96)]; + $fields = ['name' => substr($name, 0, 96), 'url' => '']; if (!empty($url) && ($url != $name)) { $fields['url'] = strtolower($url); @@ -163,4 +163,48 @@ class Tag self::storeByHash($uriid, $tag[1], $tag[3], $tag[2]); } } + + /** + * Remove tag/mention + * + * @param integer $uriid + * @param integer $type + * @param string $name + * @param string $url + */ + public static function remove(int $uriid, int $type, string $name, string $url = '') + { + $tag = DBA::fetchFirst("SELECT `id` FROM `tag` INNER JOIN `post-tag` ON `post-tag`.`tid` = `tag`.`id` + WHERE `uri-id` = ? AND `type` = ? AND `name` = ? AND `url` = ?", $uriid, $type, $name, $url); + if (!DBA::isResult($tag)) { + return; + } + Logger::info('Removing tag/mention', ['uri-id' => $uriid, 'tid' => $tag['id'], 'name' => $name, 'url' => $url]); + DBA::delete('post-tag', ['uri-id' => $uriid, 'tid' => $tag['id']]); + } + + /** + * Remove tag/mention + * + * @param integer $uriid + * @param string $hash + * @param string $name + * @param string $url + */ + public static function removeByHash(int $uriid, string $hash, string $name, string $url = '') + { + if ($hash == self::TAG_CHARACTER[self::MENTION]) { + $type = self::MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]) { + $type = self::EXCLUSIVE_MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::IMPLICIT_MENTION]) { + $type = self::IMPLICIT_MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::HASHTAG]) { + $type = self::HASHTAG; + } else { + return; + } + + self::remove($uriid, $type, $name, $url); + } } From ea60660c6d3962b69020643bb52fa0fa9d1295c0 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 18 Apr 2020 21:01:43 +0000 Subject: [PATCH 33/43] Central function to fetch the type for a given hash --- src/Model/Tag.php | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index b9e04ddd7a..f11286e6a3 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -127,15 +127,8 @@ class Tag */ public static function storeByHash(int $uriid, string $hash, string $name, string $url = '') { - if ($hash == self::TAG_CHARACTER[self::MENTION]) { - $type = self::MENTION; - } elseif ($hash == self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]) { - $type = self::EXCLUSIVE_MENTION; - } elseif ($hash == self::TAG_CHARACTER[self::IMPLICIT_MENTION]) { - $type = self::IMPLICIT_MENTION; - } elseif ($hash == self::TAG_CHARACTER[self::HASHTAG]) { - $type = self::HASHTAG; - } else { + $type = self::getTypeForHash($hash); + if ($type == self::UNKNOWN) { return; } @@ -193,18 +186,33 @@ class Tag */ public static function removeByHash(int $uriid, string $hash, string $name, string $url = '') { - if ($hash == self::TAG_CHARACTER[self::MENTION]) { - $type = self::MENTION; - } elseif ($hash == self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]) { - $type = self::EXCLUSIVE_MENTION; - } elseif ($hash == self::TAG_CHARACTER[self::IMPLICIT_MENTION]) { - $type = self::IMPLICIT_MENTION; - } elseif ($hash == self::TAG_CHARACTER[self::HASHTAG]) { - $type = self::HASHTAG; - } else { + $type = self::getTypeForHash($hash); + if ($type == self::UNKNOWN) { return; } self::remove($uriid, $type, $name, $url); } + + /** + * Get the type for the given hash + * + * @param string $hash + * @return integer type + */ + private static function getTypeForHash(string $hash) + { + if ($hash == self::TAG_CHARACTER[self::MENTION]) { + return self::MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]) { + return self::EXCLUSIVE_MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::IMPLICIT_MENTION]) { + return self::IMPLICIT_MENTION; + } elseif ($hash == self::TAG_CHARACTER[self::HASHTAG]) { + return self::HASHTAG; + } else { + return self::UNKNOWN; + } + + } } From 66f5e7c0f883b08d6c8dd7fef4f95259ca874939 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 19 Apr 2020 07:24:36 +0000 Subject: [PATCH 34/43] Don't store URL with hashtags / OStatus-Diaspora-Improvements --- src/Model/Tag.php | 2 +- src/Protocol/Diaspora.php | 2 ++ src/Protocol/OStatus.php | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index f11286e6a3..bd1945c480 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -94,7 +94,7 @@ class Tag if (empty($cid)) { $fields = ['name' => substr($name, 0, 96), 'url' => '']; - if (!empty($url) && ($url != $name)) { + if (($type != Tag::HASHTAG) && !empty($url) && ($url != $name)) { $fields['url'] = strtolower($url); } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index a8ffcbcfe5..d7e8f60a2e 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -2781,6 +2781,8 @@ class Diaspora $datarray["body"] = $prefix.$original_item["body"]."[/share]"; + Tag::storeFromBody($datarray['uri-id'], $datarray["body"]); + $datarray["tag"] = $original_item["tag"]; $datarray["attach"] = $original_item["attach"]; $datarray["app"] = $original_item["app"]; diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index d95810e3a6..ccec6d9348 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -709,6 +709,8 @@ class OStatus $item["body"] = add_page_info_to_body($item["body"]); } + Tag::storeFromBody($item['uri-id'], $item['body']); + // Mastodon Content Warning if (($item["verb"] == Activity::POST) && $xpath->evaluate('boolean(atom:summary)', $entry)) { $clear_text = XML::getFirstNodeValue($xpath, 'atom:summary/text()', $entry); From 98b3058601054fcb9f330a6990745fef8f1c3e1f Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 19 Apr 2020 08:34:20 +0000 Subject: [PATCH 35/43] Update item-uri with guid if given --- src/Model/ItemURI.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Model/ItemURI.php b/src/Model/ItemURI.php index 265585be7b..12e8d915df 100644 --- a/src/Model/ItemURI.php +++ b/src/Model/ItemURI.php @@ -42,13 +42,17 @@ class ItemURI DBA::insert('item-uri', $fields, true); } - $itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $uri]); + $itemuri = DBA::selectFirst('item-uri', ['id', 'guid'], ['uri' => $uri]); if (!DBA::isResult($itemuri)) { // This shouldn't happen return null; } + if (empty($itemuri['guid']) && !empty($fields['guid'])) { + DBA::update('item-uri', ['guid' => $fields['guid']], ['id' => $itemuri['id']]); + } + return $itemuri['id']; } From 538e212a84873dfa290c594e66a8c038fbf5b382 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 19 Apr 2020 16:33:06 +0000 Subject: [PATCH 36/43] Hashtag handling with Diaspora improved --- src/Content/Text/BBCode.php | 2 +- src/Model/Item.php | 5 ++++- src/Model/Tag.php | 34 ++++++++++++++++++++++++++++++++++ src/Protocol/Diaspora.php | 4 ++-- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index 334efb32a7..601bf63b25 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -2100,7 +2100,7 @@ class BBCode $ret = []; // Convert hashtag links to hashtags - $string = preg_replace('/#\[url\=([^\[\]]*)\](.*?)\[\/url\]/ism', '#$2', $string); + $string = preg_replace('/#\[url\=([^\[\]]*)\](.*?)\[\/url\]/ism', '#$2 ', $string); // ignore anything in a code block $string = preg_replace('/\[code.*?\].*?\[\/code\]/sm', '', $string); diff --git a/src/Model/Item.php b/src/Model/Item.php index 1f2faa2698..0e82189298 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -2610,7 +2610,10 @@ class Item // This sorting is important when there are hashtags that are part of other hashtags // Otherwise there could be problems with hashtags like #test and #test2 - rsort($tags); + // Because of this we are sorting from the longest to the shortest tag. + usort($rawtags, function($a, $b) { + return strlen($b) <=> strlen($a); + }); $URLSearchString = "^\[\]"; diff --git a/src/Model/Tag.php b/src/Model/Tag.php index bd1945c480..ac26009033 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -21,7 +21,9 @@ namespace Friendica\Model; +use Friendica\Content\Text\BBCode; use Friendica\Core\Logger; +use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\Util\Strings; @@ -87,6 +89,7 @@ class Tag } else { // The contact wasn't found in the system (most likely some dead account) // We ensure that we only store a single entry by overwriting the previous name + Logger::info('Update tag', ['url' => $url, 'name' => $name]); DBA::update('tag', ['name' => substr($name, 0, 96)], ['url' => $url]); } } @@ -148,15 +151,46 @@ class Tag $tags = self::TAG_CHARACTER[self::HASHTAG] . self::TAG_CHARACTER[self::MENTION] . self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]; } + Logger::info('Check for tags', ['uri-id' => $uriid, 'hash' => $tags, 'callstack' => System::callstack()]); + if (!preg_match_all("/([" . $tags . "])\[url\=([^\[\]]*)\]([^\[\]]*)\[\/url\]/ism", $body, $result, PREG_SET_ORDER)) { return; } + Logger::info('Found tags', ['uri-id' => $uriid, 'hash' => $tags, 'result' => $result]); + foreach ($result as $tag) { self::storeByHash($uriid, $tag[1], $tag[3], $tag[2]); } } + /** + * Store raw tags (not encapsulated in links) from the body + * This function is needed in the intermediate phase. + * Later we can call item::setHashtags in advance to have all tags converted. + * + * @param integer $uriid URI-Id + * @param string $body Body of the post + */ + public static function storeRawTagsFromBody(int $uriid, string $body) + { + Logger::info('Check for tags', ['uri-id' => $uriid, 'callstack' => System::callstack()]); + + $result = BBCode::getTags($body); + if (empty($result)) { + return; + } + + Logger::info('Found tags', ['uri-id' => $uriid, 'result' => $result]); + + foreach ($result as $tag) { + if (substr($tag, 0, 1) != self::TAG_CHARACTER[self::HASHTAG]) { + continue; + } + self::storeByHash($uriid, substr($tag, 0, 1), substr($tag, 1)); + } + } + /** * Remove tag/mention * diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index d7e8f60a2e..513cd0bcc0 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1938,7 +1938,7 @@ class Diaspora $datarray["body"] = self::replacePeopleGuid($body, $person["url"]); self::storeMentions($datarray['uri-id'], $text); - Tag::storeFromBody($datarray['uri-id'], $datarray["body"], '#'); + Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]); self::fetchGuid($datarray); @@ -3015,7 +3015,7 @@ class Diaspora $datarray["body"] = self::replacePeopleGuid($body, $contact["url"]); self::storeMentions($datarray['uri-id'], $text); - Tag::storeFromBody($datarray['uri-id'], $datarray["body"], '#'); + Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]); if ($provider_display_name != "") { $datarray["app"] = $provider_display_name; From 87aebe6af5d018e26190fd37a1ab2f2a4b219ebb Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 19 Apr 2020 16:43:37 +0000 Subject: [PATCH 37/43] Default value --- static/dbstructure.config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index d6bd418f8a..4eabe2d37b 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1310,7 +1310,7 @@ return [ "fields" => [ "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""], - "tid" => ["type" => "int unsigned", "not null" => "1", "relation" => ["tag" => "id"], "primary" => "1", "comment" => ""], + "tid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["tag" => "id"], "comment" => ""], "cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["contact" => "id"], "comment" => "Contact id of the mentioned public contact"], ], "indexes" => [ From 2bb76d96d6a287b11904634ad476f0980433aebe Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 19 Apr 2020 16:48:03 +0000 Subject: [PATCH 38/43] Improved logging --- src/Model/Tag.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index ac26009033..7ef4a2d82e 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -117,7 +117,7 @@ class Tag DBA::insert('post-tag', ['uri-id' => $uriid, 'type' => $type, 'tid' => $tagid, 'cid' => $cid], true); - Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid]); + Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'callstack' => System::callstack(8)]); } /** @@ -206,7 +206,7 @@ class Tag if (!DBA::isResult($tag)) { return; } - Logger::info('Removing tag/mention', ['uri-id' => $uriid, 'tid' => $tag['id'], 'name' => $name, 'url' => $url]); + Logger::info('Removing tag/mention', ['uri-id' => $uriid, 'tid' => $tag['id'], 'name' => $name, 'url' => $url, 'callstack' => System::callstack(8)]); DBA::delete('post-tag', ['uri-id' => $uriid, 'tid' => $tag['id']]); } From f4543b2cf758ad1fe748df0256f6ef31cd4ba01b Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Apr 2020 05:42:46 +0000 Subject: [PATCH 39/43] Fix: Wrong variable --- src/Model/Item.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index 0e82189298..f6d08047b2 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -2611,7 +2611,7 @@ class Item // This sorting is important when there are hashtags that are part of other hashtags // Otherwise there could be problems with hashtags like #test and #test2 // Because of this we are sorting from the longest to the shortest tag. - usort($rawtags, function($a, $b) { + usort($tags, function($a, $b) { return strlen($b) <=> strlen($a); }); From 21103a57355b7fa7c91d8d45d72bd0f3fd6abe13 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Apr 2020 05:43:13 +0000 Subject: [PATCH 40/43] Store implicit mentions --- src/Model/Tag.php | 13 ++++++++++++- src/Protocol/ActivityPub/Processor.php | 11 +++++------ src/Protocol/DFRN.php | 4 ++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 7ef4a2d82e..f99a83daef 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -115,7 +115,18 @@ class Tag } } - DBA::insert('post-tag', ['uri-id' => $uriid, 'type' => $type, 'tid' => $tagid, 'cid' => $cid], true); + $fields = ['uri-id' => $uriid, 'type' => $type, 'tid' => $tagid, 'cid' => $cid]; + + if (in_array($type, [Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION])) { + $condition = $fields; + $condition['type'] = [Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION]; + if (DBA::exists('post-tag', $condition)) { + Logger::info('Tag already exists', $fields); + return; + } + } + + DBA::insert('post-tag', $fields, true); Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'callstack' => System::callstack(8)]); } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index b770e8a0de..2c0176257f 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -407,6 +407,7 @@ class Processor $item['tag'] = self::constructTagString($activity['tags'], $activity['sensitive']); + Tag::storeFromBody($item['uri-id'], $item['body'], '@!'); self::storeTags($item['uri-id'], $activity['tags']); $item['location'] = $activity['location']; @@ -602,9 +603,8 @@ class Processor Tag::TAG_CHARACTER[Tag::EXCLUSIVE_MENTION], Tag::TAG_CHARACTER[Tag::IMPLICIT_MENTION]])) { $tag['name'] = substr($tag['name'], 1); - } else { - $hash = '#'; } + $type = Tag::IMPLICIT_MENTION; if (!empty($tag['href'])) { $apcontact = APContact::getByURL($tag['href']); @@ -613,18 +613,17 @@ class Processor } } } elseif ($tag['type'] == 'Hashtag') { - if (substr($tag['name'], 0, 1) == Term::TAG_CHARACTER[Term::HASHTAG]) { + if ($hash == Tag::TAG_CHARACTER[Tag::HASHTAG]) { $tag['name'] = substr($tag['name'], 1); - } else { - $hash = '@'; } + $type = Tag::HASHTAG; } if (empty($tag['name'])) { continue; } - Tag::storeByHash($uriid, $hash, $tag['name'], $tag['href']); + Tag::store($uriid, $type, $tag['name'], $tag['href']); } } diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 653e5c242b..83010f811e 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -2410,7 +2410,7 @@ class DFRN $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]); - Tag::storeFromBody($item['uri-id'], $item["body"], '#'); + Tag::storeFromBody($item['uri-id'], $item["body"]); // We store the data from "dfrn:diaspora_signature" in a different table, this is done in "Item::insert" $dsprsig = XML::unescape(XML::getFirstNodeValue($xpath, "dfrn:diaspora_signature/text()", $entry)); @@ -2466,7 +2466,7 @@ class DFRN $item["tag"] .= $termhash . "[url=" . $termurl . "]" . $term . "[/url]"; - Tag::storeByHash($item['uri-id'], $termhash, $term, $termurl); + Tag::store($item['uri-id'], Tag::IMPLICIT_MENTION, $term, $termurl); } } } From 27ea747e99c2d0e0b952a56850fedeacde1a404b Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Apr 2020 09:47:26 +0000 Subject: [PATCH 41/43] Switchable contact probing --- src/Model/Tag.php | 57 +++++++++++++++++--------- src/Protocol/ActivityPub/Processor.php | 15 ++++--- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index f99a83daef..e2ce5ad0d5 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -61,10 +61,11 @@ class Tag * * @param integer $uriid * @param integer $type - * @param string $name - * @param string $url + * @param string $name + * @param string $url + * @param boolean $probing */ - public static function store(int $uriid, int $type, string $name, string $url = '') + public static function store(int $uriid, int $type, string $name, string $url = '', $probing = true) { $name = trim($name, "\x00..\x20\xFF#!@"); if (empty($name)) { @@ -80,16 +81,32 @@ class Tag return; } - Logger::info('Get ID for contact', ['url' => $url]); + if (!$probing) { + $condition = ['nurl' => Strings::normaliseLink($url), 'uid' => 0, 'deleted' => false]; + $contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]); + if (DBA::isResult($contact)) { + $cid = $contact['id']; + Logger::info('Got id for contact url', ['cid' => $cid, 'url' => $url]); + } - $condition = ['nurl' => Strings::normaliseLink($url), 'uid' => 0, 'deleted' => false]; - $contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]); - if (DBA::isResult($contact)) { - $cid = $contact['id']; + if (empty($cid)) { + $ssl_url = str_replace('http://', 'https://', $url); + $condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $url, Strings::normaliseLink($url), $ssl_url, 0]; + $contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]); + if (DBA::isResult($contact)) { + $cid = $contact['id']; + Logger::info('Got id for contact alias', ['cid' => $cid, 'url' => $url]); + } + } } else { + $cid = Contact::getIdForURL($url, 0, true); + Logger::info('Got id by probing', ['cid' => $cid, 'url' => $url]); + } + + if (empty($cid)) { // The contact wasn't found in the system (most likely some dead account) // We ensure that we only store a single entry by overwriting the previous name - Logger::info('Update tag', ['url' => $url, 'name' => $name]); + Logger::info('Contact not found, updating tag', ['url' => $url, 'name' => $name]); DBA::update('tag', ['name' => substr($name, 0, 96)], ['url' => $url]); } } @@ -100,7 +117,7 @@ class Tag if (($type != Tag::HASHTAG) && !empty($url) && ($url != $name)) { $fields['url'] = strtolower($url); } - + $tag = DBA::selectFirst('tag', ['id'], $fields); if (!DBA::isResult($tag)) { DBA::insert('tag', $fields, true); @@ -108,7 +125,7 @@ class Tag } else { $tagid = $tag['id']; } - + if (empty($tagid)) { Logger::error('No tag id created', $fields); return; @@ -128,7 +145,7 @@ class Tag DBA::insert('post-tag', $fields, true); - Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'callstack' => System::callstack(8)]); + Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'name' => $name, 'type' => $type, 'callstack' => System::callstack(8)]); } /** @@ -138,25 +155,27 @@ class Tag * @param string $hash * @param string $name * @param string $url + * @param boolean $probing */ - public static function storeByHash(int $uriid, string $hash, string $name, string $url = '') + public static function storeByHash(int $uriid, string $hash, string $name, string $url = '', $probing = true) { $type = self::getTypeForHash($hash); if ($type == self::UNKNOWN) { return; } - self::store($uriid, $type, $name, $url); + self::store($uriid, $type, $name, $url, $probing); } /** * Store tags and mentions from the body * - * @param integer $uriid URI-Id - * @param string $body Body of the post - * @param string $tags Accepted tags + * @param integer $uriid URI-Id + * @param string $body Body of the post + * @param string $tags Accepted tags + * @param boolean $probing Perform a probing for contacts, adding them if needed */ - public static function storeFromBody(int $uriid, string $body, string $tags = null) + public static function storeFromBody(int $uriid, string $body, string $tags = null, $probing = true) { if (is_null($tags)) { $tags = self::TAG_CHARACTER[self::HASHTAG] . self::TAG_CHARACTER[self::MENTION] . self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]; @@ -171,7 +190,7 @@ class Tag Logger::info('Found tags', ['uri-id' => $uriid, 'hash' => $tags, 'result' => $result]); foreach ($result as $tag) { - self::storeByHash($uriid, $tag[1], $tag[3], $tag[2]); + self::storeByHash($uriid, $tag[1], $tag[3], $tag[2], $probing); } } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 2c0176257f..d29b6977b6 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -407,7 +407,7 @@ class Processor $item['tag'] = self::constructTagString($activity['tags'], $activity['sensitive']); - Tag::storeFromBody($item['uri-id'], $item['body'], '@!'); + self::storeFromBody($item); self::storeTags($item['uri-id'], $activity['tags']); $item['location'] = $activity['location']; @@ -421,6 +421,14 @@ class Processor return $item; } + private static function storeFromBody($item) + { + // Make sure to delete all existing tags (can happen when called via the update functionality) + DBA::delete('post-tag', ['uri-id' => $uriid]); + + Tag::storeFromBody($item['uri-id'], $item['body'], '@!'); + } + /** * Generate a GUID out of an URL * @@ -588,9 +596,6 @@ class Processor */ private static function storeTags(int $uriid, array $tags = null) { - // Make sure to delete all existing tags (can happen when called via the update functionality) - DBA::delete('post-tag', ['uri-id' => $uriid]); - foreach ($tags as $tag) { if (empty($tag['name']) || empty($tag['type']) || !in_array($tag['type'], ['Mention', 'Hashtag'])) { continue; @@ -622,7 +627,7 @@ class Processor if (empty($tag['name'])) { continue; } - + Tag::store($uriid, $type, $tag['name'], $tag['href']); } } From e19b1800a4d79e79f2880954e5016dc2efd7483b Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Apr 2020 12:19:26 +0000 Subject: [PATCH 42/43] Added documentation --- src/Protocol/ActivityPub/Processor.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index d29b6977b6..4b49bd55cf 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -421,10 +421,15 @@ class Processor return $item; } - private static function storeFromBody($item) + /** + * Store hashtags and mentions + * + * @param array $item + */ + private static function storeFromBody(array $item) { // Make sure to delete all existing tags (can happen when called via the update functionality) - DBA::delete('post-tag', ['uri-id' => $uriid]); + DBA::delete('post-tag', ['uri-id' => $item['uri-id']]); Tag::storeFromBody($item['uri-id'], $item['body'], '@!'); } From 1f7993acad370f87c899306a08e42a7693c13452 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 22 Apr 2020 15:22:39 +0000 Subject: [PATCH 43/43] Code standards --- include/items.php | 4 ++-- src/Model/UserItem.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/items.php b/include/items.php index 2d73308582..582fbb0933 100644 --- a/include/items.php +++ b/include/items.php @@ -144,7 +144,7 @@ function query_page_info($url, $photo = "", $keywords = false, $keyword_blacklis function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "") { $data = query_page_info($url, $photo, $keywords, $keyword_blacklist); - if (empty($data['keywords']) || !is_array($data['keywords'])) { + if (empty($data["keywords"]) || !is_array($data["keywords"])) { return ''; } @@ -166,7 +166,7 @@ function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blackl function get_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "") { $data = query_page_info($url, $photo, $keywords, $keyword_blacklist); - if (empty($data['keywords']) || !is_array($data['keywords'])) { + if (empty($data["keywords"]) || !is_array($data["keywords"])) { return []; } diff --git a/src/Model/UserItem.php b/src/Model/UserItem.php index d38ed6d55e..6a228c098a 100644 --- a/src/Model/UserItem.php +++ b/src/Model/UserItem.php @@ -207,7 +207,7 @@ class UserItem } // Or the contact is a mentioned forum - $tags = DBA::select('term', ['url'], ['otype' => term::OBJECT_TYPE_POST, 'oid' => $item['id'], 'type' => Term::MENTION, 'uid' => $uid]); + $tags = DBA::select('term', ['url'], ['otype' => Term::OBJECT_TYPE_POST, 'oid' => $item['id'], 'type' => Term::MENTION, 'uid' => $uid]); while ($tag = DBA::fetch($tags)) { $condition = ['nurl' => Strings::normaliseLink($tag['url']), 'uid' => $uid, 'notify_new_posts' => true, 'contact-type' => Contact::TYPE_COMMUNITY]; if (DBA::exists('contact', $condition)) {