bitPickups/friendica_2019-12_sharedHosting_hotFix
1
1
Fork 0
Code Pull requests Activity

Merge pull request #8520 from annando/term2tag

Browse source 
We now store tags in "tag"
This commit is contained in:
Hypolite Petovan 2020-04-22 22:56:07 -04:00 committed by GitHub
parent 29dfac2345 1f7993acad
commit 43b8bdea07
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 561 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
boot.php
View file

@ -178,29 +178,6 @@ define('NOTIFY_SHARE', Notify\Type::SHARE);
define('NOTIFY_SYSTEM', Notify\Type::SYSTEM);
/* @}*/
/** @deprecated since 2019.03, use Term::UNKNOWN instead */
define('TERM_UNKNOWN', Term::UNKNOWN);
/** @deprecated since 2019.03, use Term::HASHTAG instead */
define('TERM_HASHTAG', Term::HASHTAG);
/** @deprecated since 2019.03, use Term::MENTION instead */
define('TERM_MENTION', Term::MENTION);
/** @deprecated since 2019.03, use Term::CATEGORY instead */
define('TERM_CATEGORY', Term::CATEGORY);
/** @deprecated since 2019.03, use Term::PCATEGORY instead */
define('TERM_PCATEGORY', Term::PCATEGORY);
/** @deprecated since 2019.03, use Term::FILE instead */
define('TERM_FILE', Term::FILE);
/** @deprecated since 2019.03, use Term::SAVEDSEARCH instead */
define('TERM_SAVEDSEARCH', Term::SAVEDSEARCH);
/** @deprecated since 2019.03, use Term::CONVERSATION instead */
define('TERM_CONVERSATION', Term::CONVERSATION);
/** @deprecated since 2019.03, use Term::OBJECT_TYPE_POST instead */
define('TERM_OBJ_POST', Term::OBJECT_TYPE_POST);
/** @deprecated since 2019.03, use Term::OBJECT_TYPE_PHOTO instead */
define('TERM_OBJ_PHOTO', Term::OBJECT_TYPE_PHOTO);
/**
* @name Gravity
*

7
include/api.php
View file

@ -41,6 +41,7 @@ use Friendica\Model\Item;
use Friendica\Model\Mail;
use Friendica\Model\Notify;
use Friendica\Model\Photo;
use Friendica\Model\Term;
use Friendica\Model\User;
use Friendica\Model\UserItem;
use Friendica\Network\FKOAuth1;
@ -1541,7 +1542,7 @@ function api_search($type)
$condition = ["`oid` > ?
AND (`uid` = 0 OR (`uid` = ? AND NOT `global`))
AND `otype` = ? AND `type` = ? AND `term` = ?",
$since_id, local_user(), TERM_OBJ_POST, TERM_HASHTAG, $searchTerm];
$since_id, local_user(), Term::OBJECT_TYPE_POST, Term::HASHTAG, $searchTerm];
if ($max_id > 0) {
$condition[0] .= ' AND `oid` <= ?';
$condition[] = $max_id;
@ -2040,7 +2041,7 @@ function api_statuses_repeat($type)
Logger::log('API: api_statuses_repeat: '.$id);
$fields = ['body', 'title', 'attach', 'tag', 'author-name', 'author-link', 'author-avatar', 'guid', 'created', 'plink'];
$fields = ['uri-id', 'body', 'title', 'attach', 'tag', 'author-name', 'author-link', 'author-avatar', 'guid', 'created', 'plink'];
$item = Item::selectFirst($fields, ['id' => $id, 'private' => [Item::PUBLIC, Item::UNLISTED]]);
if (DBA::isResult($item) && $item['body'] != "") {
@ -2068,6 +2069,8 @@ function api_statuses_repeat($type)
}
$item_id = item_post($a);
/// @todo Copy tags from the original post to the new one
} else {
throw new ForbiddenException();
}

23
include/items.php
View file

@ -144,9 +144,11 @@ function query_page_info($url, $photo = "", $keywords = false, $keyword_blacklis
function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "")
{
$data = query_page_info($url, $photo, $keywords, $keyword_blacklist);
if (empty($data["keywords"]) || !is_array($data["keywords"])) {
return '';
}
$tags = "";
if (isset($data["keywords"]) && count($data["keywords"])) {
foreach ($data["keywords"] as $keyword) {
$hashtag = str_replace([" ", "+", "/", ".", "#", "'"],
["", "", "", "", "", ""], $keyword);
@ -157,11 +159,28 @@ function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blackl
$tags .= "#[url=" . DI::baseUrl() . "/search?tag=" . $hashtag . "]" . $hashtag . "[/url]";
}
}
return $tags;
}
function get_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "")
{
$data = query_page_info($url, $photo, $keywords, $keyword_blacklist);
if (empty($data["keywords"]) || !is_array($data["keywords"])) {
return [];
}
$taglist = [];
foreach ($data['keywords'] as $keyword) {
$hashtag = str_replace([" ", "+", "/", ".", "#", "'"],
["", "", "", "", "", ""], $keyword);
$taglist[] = $hashtag;
}
return $taglist;
}
function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "")
{
$data = query_page_info($url, $photo, $keywords, $keyword_blacklist);

3
mod/item.php
View file

@ -46,6 +46,7 @@ use Friendica\Model\FileTag;
use Friendica\Model\Item;
use Friendica\Model\Notify\Type;
use Friendica\Model\Photo;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Network\HTTPException;
use Friendica\Object\EMail\ItemCCEMail;
@ -750,6 +751,8 @@ function item_post(App $a) {
throw new HTTPException\InternalServerErrorException(DI::l10n()->t('Item couldn\'t be fetched.'));
}
Tag::storeFromBody($datarray['uri-id'], $datarray['body']);
// update filetags in pconfig
FileTag::updatePconfig($uid, $categories_old, $categories_new, 'category');

2
mod/network.php
View file

@ -793,7 +793,7 @@ function networkThreadedView(App $a, $update, $parent)
STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = `item`.`author-id`
WHERE `item`.`uid` = 0 AND `item`.$ordering < ? AND `item`.$ordering > ? AND `item`.`gravity` = ?
AND NOT `author`.`hidden` AND NOT `author`.`blocked`" . $sql_tag_nets,
local_user(), TERM_OBJ_POST, TERM_HASHTAG,
local_user(), Term::OBJECT_TYPE_POST, Term::HASHTAG,
$top_limit, $bottom_limit, GRAVITY_PARENT);
$data = DBA::toArray($items);

10
mod/photos.php
View file

@ -36,6 +36,7 @@ use Friendica\Model\Contact;
use Friendica\Model\Item;
use Friendica\Model\Photo;
use Friendica\Model\Profile;
use Friendica\Model\Tag;
use Friendica\Model\User;
use Friendica\Module\BaseProfile;
use Friendica\Network\Probe;
@ -421,7 +422,7 @@ function photos_post(App $a)
}
if ($item_id) {
$item = Item::selectFirst(['tag', 'inform'], ['id' => $item_id, 'uid' => $page_owner_uid]);
$item = Item::selectFirst(['tag', 'inform', 'uri-id'], ['id' => $item_id, 'uid' => $page_owner_uid]);
if (DBA::isResult($item)) {
$old_tag = $item['tag'];
@ -521,10 +522,17 @@ function photos_post(App $a)
$profile = str_replace(',', '%2c', $profile);
$str_tags .= '@[url=' . $profile . ']' . $newname . '[/url]';
if (!empty($item['uri-id'])) {
Tag::store($item['uri-id'], Tag::MENTION, $newname, $profile);
}
}
} elseif (strpos($tag, '#') === 0) {
$tagname = substr($tag, 1);
$str_tags .= '#[url=' . DI::baseUrl() . "/search?tag=" . $tagname . ']' . $tagname . '[/url],';
if (!empty($item['uri-id'])) {
Tag::store($item['uri-id'], Tag::HASHTAG, $tagname);
}
}
}
}

10
mod/tagger.php
View file

@ -28,6 +28,8 @@ use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Item;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Protocol\Activity;
use Friendica\Util\Strings;
use Friendica\Util\XML;
@ -168,7 +170,9 @@ EOT;
Item::update(['visible' => true], ['id' => $item['id']]);
}
$term_objtype = ($item['resource-id'] ? TERM_OBJ_PHOTO : TERM_OBJ_POST);
$term_objtype = ($item['resource-id'] ? Term::OBJECT_TYPE_PHOTO : Term::OBJECT_TYPE_POST);
Tag::store($item['uri-id'], Tag::HASHTAG, $term);
$t = q("SELECT count(tid) as tcount FROM term WHERE oid=%d AND term='%s'",
intval($item['id']),
@ -179,7 +183,7 @@ EOT;
q("INSERT INTO term (oid, otype, type, term, url, uid) VALUE (%d, %d, %d, '%s', '%s', %d)",
intval($item['id']),
$term_objtype,
TERM_HASHTAG,
Term::HASHTAG,
DBA::escape($term),
'',
intval($owner_uid)
@ -201,7 +205,7 @@ EOT;
q("INSERT INTO term (`oid`, `otype`, `type`, `term`, `url`, `uid`) VALUE (%d, %d, %d, '%s', '%s', %d)",
intval($original_item['id']),
$term_objtype,
TERM_HASHTAG,
Term::HASHTAG,
DBA::escape($term),
'',
intval($owner_uid)

9
mod/tagrm.php
View file

@ -24,6 +24,7 @@ use Friendica\Content\Text\BBCode;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Item;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Util\Strings;
@ -62,7 +63,7 @@ function update_tags($item_id, $tags){
return;
}
$item = Item::selectFirst(['tag'], ['id' => $item_id, 'uid' => local_user()]);
$item = Item::selectFirst(['tag', 'uri-id'], ['id' => $item_id, 'uid' => local_user()]);
if (!DBA::isResult($item)) {
return;
}
@ -70,6 +71,12 @@ function update_tags($item_id, $tags){
$old_tags = explode(',', $item['tag']);
foreach ($tags as $new_tag) {
if (preg_match_all('/([#@!])\[url\=([^\[\]]*)\]([^\[\]]*)\[\/url\]/ism', $new_tag, $results, PREG_SET_ORDER)) {
foreach ($results as $tag) {
Tag::removeByHash($item['uri-id'], $tag[1], $tag[3], $tag[2]);
}
}
foreach ($old_tags as $index => $old_tag) {
if (strcmp($old_tag, $new_tag) == 0) {
unset($old_tags[$index]);

7
src/Content/Widget/TagCloud.php
View file

@ -25,6 +25,7 @@ use Friendica\Core\Renderer;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Item;
use Friendica\Model\Term;
/**
* TagCloud widget
@ -45,7 +46,7 @@ class TagCloud
* @return string HTML formatted output.
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
public static function getHTML($uid, $count = 0, $owner_id = 0, $flags = '', $type = TERM_HASHTAG)
public static function getHTML($uid, $count = 0, $owner_id = 0, $flags = '', $type = Term::HASHTAG)
{
$o = '';
$r = self::tagadelic($uid, $count, $owner_id, $flags, $type);
@ -84,7 +85,7 @@ class TagCloud
* @return array Alphabetical sorted array of used tags of an user.
* @throws \Exception
*/
private static function tagadelic($uid, $count = 0, $owner_id = 0, $flags = '', $type = TERM_HASHTAG)
private static function tagadelic($uid, $count = 0, $owner_id = 0, $flags = '', $type = Term::HASHTAG)
{
$sql_options = Item::getPermissionsSQLByUserId($uid);
$limit = $count ? sprintf('LIMIT %d', intval($count)) : '';
@ -109,7 +110,7 @@ class TagCloud
GROUP BY `term` ORDER BY `total` DESC $limit",
$uid,
$type,
TERM_OBJ_POST
Term::OBJECT_TYPE_POST
);
if (!DBA::isResult($tag_stmt)) {
return [];

5
src/Model/FileTag.php
View file

@ -23,6 +23,7 @@ namespace Friendica\Model;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Term;
/**
* This class handles FileTag related functions
@ -195,11 +196,11 @@ class FileTag
if ($type == 'file') {
$lbracket = '[';
$rbracket = ']';
$termtype = TERM_FILE;
$termtype = Term::FILE;
} else {
$lbracket = '<';
$rbracket = '>';
$termtype = TERM_CATEGORY;
$termtype = Term::CATEGORY;
}
$filetags_updated = $saved;

8
src/Model/Item.php
View file

@ -94,7 +94,8 @@ class Item
const CONTENT_FIELDLIST = ['language'];
// All fields in the item table
const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid',
const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent',
'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id',
'contact-id', 'type', 'wall', 'gravity', 'extid', 'icid', 'iaid', 'psid',
'created', 'edited', 'commented', 'received', 'changed', 'verb',
'postopts', 'plink', 'resource-id', 'event-id', 'tag', 'attach', 'inform',
@ -2609,7 +2610,10 @@ class Item
// This sorting is important when there are hashtags that are part of other hashtags
// Otherwise there could be problems with hashtags like #test and #test2
rsort($tags);
// Because of this we are sorting from the longest to the shortest tag.
usort($tags, function($a, $b) {
return strlen($b) <=> strlen($a);
});
$URLSearchString = "^\[\]";

6
src/Model/ItemURI.php
View file

@ -42,13 +42,17 @@ class ItemURI
DBA::insert('item-uri', $fields, true);
}
$itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $uri]);
$itemuri = DBA::selectFirst('item-uri', ['id', 'guid'], ['uri' => $uri]);
if (!DBA::isResult($itemuri)) {
// This shouldn't happen
return null;
}
if (empty($itemuri['guid']) && !empty($fields['guid'])) {
DBA::update('item-uri', ['guid' => $fields['guid']], ['id' => $itemuri['id']]);
}
return $itemuri['id'];
}

282
src/Model/Tag.php Normal file
View file

@ -0,0 +1,282 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Model;
use Friendica\Content\Text\BBCode;
use Friendica\Core\Logger;
use Friendica\Core\System;
use Friendica\Database\DBA;
use Friendica\Util\Strings;
/**
* Class Tag
*
* This Model class handles tag table interactions.
* This tables stores relevant tags related to posts, like hashtags and mentions.
*/
class Tag
{
const UNKNOWN = 0;
const HASHTAG = 1;
const MENTION = 2;
const CATEGORY = 3;
const FILE = 5;
/**
* An implicit mention is a mention in a comment body that is redundant with the threading information.
*/
const IMPLICIT_MENTION = 8;
/**
* An exclusive mention transfers the ownership of the post to the target account, usually a forum.
*/
const EXCLUSIVE_MENTION = 9;
const TAG_CHARACTER = [
self::HASHTAG => '#',
self::MENTION => '@',
self::IMPLICIT_MENTION => '%',
self::EXCLUSIVE_MENTION => '!',
];
/**
* Store tag/mention elements
*
* @param integer $uriid
* @param integer $type
* @param string $name
* @param string $url
* @param boolean $probing
*/
public static function store(int $uriid, int $type, string $name, string $url = '', $probing = true)
{
$name = trim($name, "\x00..\x20\xFF#!@");
if (empty($name)) {
return;
}
$cid = 0;
$tagid = 0;
if (in_array($type, [Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION])) {
if (empty($url)) {
// No mention without a contact url
return;
}
if (!$probing) {
$condition = ['nurl' => Strings::normaliseLink($url), 'uid' => 0, 'deleted' => false];
$contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]);
if (DBA::isResult($contact)) {
$cid = $contact['id'];
Logger::info('Got id for contact url', ['cid' => $cid, 'url' => $url]);
}
if (empty($cid)) {
$ssl_url = str_replace('http://', 'https://', $url);
$condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $url, Strings::normaliseLink($url), $ssl_url, 0];
$contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]);
if (DBA::isResult($contact)) {
$cid = $contact['id'];
Logger::info('Got id for contact alias', ['cid' => $cid, 'url' => $url]);
}
}
} else {
$cid = Contact::getIdForURL($url, 0, true);
Logger::info('Got id by probing', ['cid' => $cid, 'url' => $url]);
}
if (empty($cid)) {
// The contact wasn't found in the system (most likely some dead account)
// We ensure that we only store a single entry by overwriting the previous name
Logger::info('Contact not found, updating tag', ['url' => $url, 'name' => $name]);
DBA::update('tag', ['name' => substr($name, 0, 96)], ['url' => $url]);
}
}
if (empty($cid)) {
$fields = ['name' => substr($name, 0, 96), 'url' => ''];
if (($type != Tag::HASHTAG) && !empty($url) && ($url != $name)) {
$fields['url'] = strtolower($url);
}
$tag = DBA::selectFirst('tag', ['id'], $fields);
if (!DBA::isResult($tag)) {
DBA::insert('tag', $fields, true);
$tagid = DBA::lastInsertId();
} else {
$tagid = $tag['id'];
}
if (empty($tagid)) {
Logger::error('No tag id created', $fields);
return;
}
}
$fields = ['uri-id' => $uriid, 'type' => $type, 'tid' => $tagid, 'cid' => $cid];
if (in_array($type, [Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION])) {
$condition = $fields;
$condition['type'] = [Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION];
if (DBA::exists('post-tag', $condition)) {
Logger::info('Tag already exists', $fields);
return;
}
}
DBA::insert('post-tag', $fields, true);
Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'name' => $name, 'type' => $type, 'callstack' => System::callstack(8)]);
}
/**
* Store tag/mention elements
*
* @param integer $uriid
* @param string $hash
* @param string $name
* @param string $url
* @param boolean $probing
*/
public static function storeByHash(int $uriid, string $hash, string $name, string $url = '', $probing = true)
{
$type = self::getTypeForHash($hash);
if ($type == self::UNKNOWN) {
return;
}
self::store($uriid, $type, $name, $url, $probing);
}
/**
* Store tags and mentions from the body
*
* @param integer $uriid URI-Id
* @param string $body Body of the post
* @param string $tags Accepted tags
* @param boolean $probing Perform a probing for contacts, adding them if needed
*/
public static function storeFromBody(int $uriid, string $body, string $tags = null, $probing = true)
{
if (is_null($tags)) {
$tags = self::TAG_CHARACTER[self::HASHTAG] . self::TAG_CHARACTER[self::MENTION] . self::TAG_CHARACTER[self::EXCLUSIVE_MENTION];
}
Logger::info('Check for tags', ['uri-id' => $uriid, 'hash' => $tags, 'callstack' => System::callstack()]);
if (!preg_match_all("/([" . $tags . "])\[url\=([^\[\]]*)\]([^\[\]]*)\[\/url\]/ism", $body, $result, PREG_SET_ORDER)) {
return;
}
Logger::info('Found tags', ['uri-id' => $uriid, 'hash' => $tags, 'result' => $result]);
foreach ($result as $tag) {
self::storeByHash($uriid, $tag[1], $tag[3], $tag[2], $probing);
}
}
/**
* Store raw tags (not encapsulated in links) from the body
* This function is needed in the intermediate phase.
* Later we can call item::setHashtags in advance to have all tags converted.
*
* @param integer $uriid URI-Id
* @param string $body Body of the post
*/
public static function storeRawTagsFromBody(int $uriid, string $body)
{
Logger::info('Check for tags', ['uri-id' => $uriid, 'callstack' => System::callstack()]);
$result = BBCode::getTags($body);
if (empty($result)) {
return;
}
Logger::info('Found tags', ['uri-id' => $uriid, 'result' => $result]);
foreach ($result as $tag) {
if (substr($tag, 0, 1) != self::TAG_CHARACTER[self::HASHTAG]) {
continue;
}
self::storeByHash($uriid, substr($tag, 0, 1), substr($tag, 1));
}
}
/**
* Remove tag/mention
*
* @param integer $uriid
* @param integer $type
* @param string $name
* @param string $url
*/
public static function remove(int $uriid, int $type, string $name, string $url = '')
{
$tag = DBA::fetchFirst("SELECT `id` FROM `tag` INNER JOIN `post-tag` ON `post-tag`.`tid` = `tag`.`id`
WHERE `uri-id` = ? AND `type` = ? AND `name` = ? AND `url` = ?", $uriid, $type, $name, $url);
if (!DBA::isResult($tag)) {
return;
}
Logger::info('Removing tag/mention', ['uri-id' => $uriid, 'tid' => $tag['id'], 'name' => $name, 'url' => $url, 'callstack' => System::callstack(8)]);
DBA::delete('post-tag', ['uri-id' => $uriid, 'tid' => $tag['id']]);
}
/**
* Remove tag/mention
*
* @param integer $uriid
* @param string $hash
* @param string $name
* @param string $url
*/
public static function removeByHash(int $uriid, string $hash, string $name, string $url = '')
{
$type = self::getTypeForHash($hash);
if ($type == self::UNKNOWN) {
return;
}
self::remove($uriid, $type, $name, $url);
}
/**
* Get the type for the given hash
*
* @param string $hash
* @return integer type
*/
private static function getTypeForHash(string $hash)
{
if ($hash == self::TAG_CHARACTER[self::MENTION]) {
return self::MENTION;
} elseif ($hash == self::TAG_CHARACTER[self::EXCLUSIVE_MENTION]) {
return self::EXCLUSIVE_MENTION;
} elseif ($hash == self::TAG_CHARACTER[self::IMPLICIT_MENTION]) {
return self::IMPLICIT_MENTION;
} elseif ($hash == self::TAG_CHARACTER[self::HASHTAG]) {
return self::HASHTAG;
} else {
return self::UNKNOWN;
}
}
}

7
src/Model/Term.php
View file

@ -40,10 +40,7 @@ class Term
const HASHTAG = 1;
const MENTION = 2;
const CATEGORY = 3;
const PCATEGORY = 4;
const FILE = 5;
const SAVEDSEARCH = 6;
const CONVERSATION = 7;
/**
* An implicit mention is a mention in a comment body that is redundant with the threading information.
*/
@ -333,6 +330,10 @@ class Term
continue;
}
if (empty($term)) {
continue;
}
if ($item['uid'] == 0) {
$global = true;
DBA::update('term', ['global' => true], ['otype' => self::OBJECT_TYPE_POST, 'guid' => $item['guid']]);

3
src/Model/UserItem.php
View file

@ -26,6 +26,7 @@ use Friendica\Core\Hook;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\Strings;
use Friendica\Model\Term;
class UserItem
{
@ -206,7 +207,7 @@ class UserItem
}
// Or the contact is a mentioned forum
$tags = DBA::select('term', ['url'], ['otype' => TERM_OBJ_POST, 'oid' => $item['id'], 'type' => TERM_MENTION, 'uid' => $uid]);
$tags = DBA::select('term', ['url'], ['otype' => Term::OBJECT_TYPE_POST, 'oid' => $item['id'], 'type' => Term::MENTION, 'uid' => $uid]);
while ($tag = DBA::fetch($tags)) {
$condition = ['nurl' => Strings::normaliseLink($tag['url']), 'uid' => $uid, 'notify_new_posts' => true, 'contact-type' => Contact::TYPE_COMMUNITY];
if (DBA::exists('contact', $condition)) {

3
src/Module/Hashtag.php
View file

@ -25,6 +25,7 @@ use Friendica\BaseModule;
use Friendica\Core\System;
use Friendica\Database\DBA;
use Friendica\Util\Strings;
use Friendica\Model\Term;
/**
* Hashtag module.
@ -43,7 +44,7 @@ class Hashtag extends BaseModule
$taglist = DBA::p("SELECT DISTINCT(`term`) FROM `term` WHERE `term` LIKE ? AND `type` = ? ORDER BY `term`",
$t . '%',
intval(TERM_HASHTAG)
intval(Term::HASHTAG)
);
while ($tag = DBA::fetch($taglist)) {
$result[] = ['text' => $tag['term']];

5
src/Module/Profile/Status.php
View file

@ -31,6 +31,7 @@ use Friendica\DI;
use Friendica\Model\Item;
use Friendica\Model\Profile as ProfileModel;
use Friendica\Model\User;
use Friendica\Model\Term;
use Friendica\Module\BaseProfile;
use Friendica\Module\Security\Login;
use Friendica\Util\DateTimeFormat;
@ -142,12 +143,12 @@ class Status extends BaseProfile
if (!empty($category)) {
$sql_post_table = sprintf("INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ",
DBA::escape(Strings::protectSprintf($category)), intval(TERM_OBJ_POST), intval(TERM_CATEGORY), intval($a->profile['uid']));
DBA::escape(Strings::protectSprintf($category)), intval(Term::OBJECT_TYPE_POST), intval(Term::CATEGORY), intval($a->profile['uid']));
}
if (!empty($hashtags)) {
$sql_post_table .= sprintf("INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ",
DBA::escape(Strings::protectSprintf($hashtags)), intval(TERM_OBJ_POST), intval(TERM_HASHTAG), intval($a->profile['uid']));
DBA::escape(Strings::protectSprintf($hashtags)), intval(Term::OBJECT_TYPE_POST), intval(Term::HASHTAG), intval($a->profile['uid']));
}
if (!empty($datequery)) {

70
src/Protocol/ActivityPub/Processor.php
View file

@ -32,7 +32,9 @@ use Friendica\Model\Contact;
use Friendica\Model\Conversation;
use Friendica\Model\Event;
use Friendica\Model\Item;
use Friendica\Model\ItemURI;
use Friendica\Model\Mail;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Model\User;
use Friendica\Protocol\Activity;
@ -169,7 +171,7 @@ class Processor
*/
public static function updateItem($activity)
{
$item = Item::selectFirst(['uri', 'thr-parent', 'gravity'], ['uri' => $activity['id']]);
$item = Item::selectFirst(['uri', 'uri-id', 'thr-parent', 'gravity'], ['uri' => $activity['id']]);
if (!DBA::isResult($item)) {
Logger::warning('Unknown item', ['uri' => $activity['id']]);
return;
@ -249,7 +251,7 @@ class Processor
}
foreach ($activity['receiver'] as $receiver) {
$item = Item::selectFirst(['id', 'tag', 'origin', 'author-link'], ['uri' => $activity['target_id'], 'uid' => $receiver]);
$item = Item::selectFirst(['id', 'uri-id', 'tag', 'origin', 'author-link'], ['uri' => $activity['target_id'], 'uid' => $receiver]);
if (!DBA::isResult($item)) {
// We don't fetch missing content for this purpose
continue;
@ -260,6 +262,8 @@ class Processor
continue;
}
Tag::store($item['uri-id'], Tag::HASHTAG, $activity['object_content'], $activity['object_id']);
// To-Do:
// - Check if "blocktag" is set
// - Check if actor is a contact
@ -403,6 +407,9 @@ class Processor
$item['tag'] = self::constructTagString($activity['tags'], $activity['sensitive']);
self::storeFromBody($item);
self::storeTags($item['uri-id'], $activity['tags']);
$item['location'] = $activity['location'];
if (!empty($item['latitude']) && !empty($item['longitude'])) {
@ -414,6 +421,19 @@ class Processor
return $item;
}
/**
* Store hashtags and mentions
*
* @param array $item
*/
private static function storeFromBody(array $item)
{
// Make sure to delete all existing tags (can happen when called via the update functionality)
DBA::delete('post-tag', ['uri-id' => $item['uri-id']]);
Tag::storeFromBody($item['uri-id'], $item['body'], '@!');
}
/**
* Generate a GUID out of an URL
*
@ -496,6 +516,8 @@ class Processor
$item['edited'] = DateTimeFormat::utc($activity['updated']);
$item['guid'] = $activity['diaspora:guid'] ?: $activity['sc:identifier'] ?: self::getGUIDByURL($item['uri']);
$item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]);
$item = self::processContent($activity, $item);
if (empty($item)) {
return;
@ -571,6 +593,50 @@ class Processor
}
}
/**
* Store tags and mentions into the tag table
*
* @param integer $uriid
* @param array $tags
*/
private static function storeTags(int $uriid, array $tags = null)
{
foreach ($tags as $tag) {
if (empty($tag['name']) || empty($tag['type']) || !in_array($tag['type'], ['Mention', 'Hashtag'])) {
continue;
}
$hash = substr($tag['name'], 0, 1);
if ($tag['type'] == 'Mention') {
if (in_array($hash, [Tag::TAG_CHARACTER[Tag::MENTION],
Tag::TAG_CHARACTER[Tag::EXCLUSIVE_MENTION],
Tag::TAG_CHARACTER[Tag::IMPLICIT_MENTION]])) {
$tag['name'] = substr($tag['name'], 1);
}
$type = Tag::IMPLICIT_MENTION;
if (!empty($tag['href'])) {
$apcontact = APContact::getByURL($tag['href']);
if (!empty($apcontact['name']) || !empty($apcontact['nick'])) {
$tag['name'] = $apcontact['name'] ?: $apcontact['nick'];
}
}
} elseif ($tag['type'] == 'Hashtag') {
if ($hash == Tag::TAG_CHARACTER[Tag::HASHTAG]) {
$tag['name'] = substr($tag['name'], 1);
}
$type = Tag::HASHTAG;
}
if (empty($tag['name'])) {
continue;
}
Tag::store($uriid, $type, $tag['name'], $tag['href']);
}
}
/**
* Creates an mail post
*

2
src/Protocol/ActivityPub/Transmitter.php
View file

@ -1218,7 +1218,7 @@ class Transmitter
*/
private static function isSensitive($item_id)
{
$condition = ['otype' => TERM_OBJ_POST, 'oid' => $item_id, 'type' => TERM_HASHTAG, 'term' => 'nsfw'];
$condition = ['otype' => Term::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => Term::HASHTAG, 'term' => 'nsfw'];
return DBA::exists('term', $condition);
}

21
src/Protocol/DFRN.php
View file

@ -24,9 +24,7 @@ namespace Friendica\Protocol;
use DOMDocument;
use DOMXPath;
use Friendica\App\BaseURL;
use Friendica\Content\OEmbed;
use Friendica\Content\Text\BBCode;
use Friendica\Content\Text\HTML;
use Friendica\Core\Hook;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
@ -37,10 +35,13 @@ use Friendica\Model\Conversation;
use Friendica\Model\Event;
use Friendica\Model\GContact;
use Friendica\Model\Item;
use Friendica\Model\ItemURI;
use Friendica\Model\Mail;
use Friendica\Model\Notify\Type;
use Friendica\Model\PermissionSet;
use Friendica\Model\Profile;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Model\User;
use Friendica\Network\Probe;
use Friendica\Util\Crypto;
@ -49,8 +50,6 @@ use Friendica\Util\Images;
use Friendica\Util\Network;
use Friendica\Util\Strings;
use Friendica\Util\XML;
use HTMLPurifier;
use HTMLPurifier_Config;
/**
* This class contain functions to create and send DFRN XML files
@ -252,8 +251,8 @@ class DFRN
$sql_post_table = sprintf(
"INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ",
DBA::escape(Strings::protectSprintf($category)),
intval(TERM_OBJ_POST),
intval(TERM_CATEGORY),
intval(Term::OBJECT_TYPE_POST),
intval(Term::CATEGORY),
intval($owner_id)
);
}
@ -2244,7 +2243,7 @@ class DFRN
$xt = XML::parseString($item["target"], false);
if ($xt->type == Activity\ObjectType::NOTE) {
$item_tag = Item::selectFirst(['id', 'tag'], ['uri' => $xt->id, 'uid' => $importer["importer_uid"]]);
$item_tag = Item::selectFirst(['id', 'uri-id', 'tag'], ['uri' => $xt->id, 'uid' => $importer["importer_uid"]]);
if (!DBA::isResult($item_tag)) {
Logger::log("Query failed to execute, no result returned in " . __FUNCTION__);
@ -2253,6 +2252,8 @@ class DFRN
// extract tag, if not duplicate, add to parent item
if ($xo->content) {
Tag::store($item_tag['uri-id'], Tag::HASHTAG, $xo->content);
if (!stristr($item_tag["tag"], trim($xo->content))) {
$tag = $item_tag["tag"] . (strlen($item_tag["tag"]) ? ',' : '') . '#[url=' . $xo->id . ']'. $xo->content . '[/url]';
Item::update(['tag' => $tag], ['id' => $item_tag["id"]]);
@ -2407,6 +2408,10 @@ class DFRN
$item["guid"] = XML::getFirstNodeValue($xpath, "dfrn:diaspora_guid/text()", $entry);
$item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]);
Tag::storeFromBody($item['uri-id'], $item["body"]);
// We store the data from "dfrn:diaspora_signature" in a different table, this is done in "Item::insert"
$dsprsig = XML::unescape(XML::getFirstNodeValue($xpath, "dfrn:diaspora_signature/text()", $entry));
if ($dsprsig != "") {
@ -2460,6 +2465,8 @@ class DFRN
}
$item["tag"] .= $termhash . "[url=" . $termurl . "]" . $term . "[/url]";
Tag::store($item['uri-id'], Tag::IMPLICIT_MENTION, $term, $termurl);
}
}
}

52
src/Protocol/Diaspora.php
View file

@ -35,9 +35,12 @@ use Friendica\Model\Contact;
use Friendica\Model\Conversation;
use Friendica\Model\GContact;
use Friendica\Model\Item;
use Friendica\Model\ItemURI;
use Friendica\Model\ItemDeliveryData;
use Friendica\Model\Mail;
use Friendica\Model\Profile;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Model\User;
use Friendica\Network\Probe;
use Friendica\Util\Crypto;
@ -123,7 +126,7 @@ class Diaspora
}
// All tags of the current post
$condition = ['otype' => TERM_OBJ_POST, 'type' => TERM_HASHTAG, 'oid' => $parent['parent']];
$condition = ['otype' => Term::OBJECT_TYPE_POST, 'type' => Term::HASHTAG, 'oid' => $parent['parent']];
$tags = DBA::select('term', ['term'], $condition);
$taglist = [];
while ($tag = DBA::fetch($tags)) {
@ -1807,6 +1810,40 @@ class Diaspora
return false;
}
/**
* Store the mentions in the tag table
*
* @param integer $uriid
* @param string $text
*/
private static function storeMentions(int $uriid, string $text)
{
preg_match_all('/([@!]){(?:([^}]+?); ?)?([^} ]+)}/', $text, $matches, PREG_SET_ORDER);
if (empty($matches)) {
return;
}
/*
* Matching values for the preg match
* [1] = mention type (@ or !)
* [2] = name (optional)
* [3] = profile URL
*/
foreach ($matches as $match) {
if (empty($match)) {
continue;
}
$person = self::personByHandle($match[3]);
if (empty($person)) {
continue;
}
Tag::storeByHash($uriid, $match[1], $person['name'] ?: $person['nick'], $person['url']);
}
}
/**
* Processes an incoming comment
*
@ -1877,6 +1914,7 @@ class Diaspora
$datarray["guid"] = $guid;
$datarray["uri"] = self::getUriFromGuid($author, $guid);
$datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]);
$datarray["verb"] = Activity::POST;
$datarray["gravity"] = GRAVITY_COMMENT;
@ -1899,6 +1937,9 @@ class Diaspora
$datarray["body"] = self::replacePeopleGuid($body, $person["url"]);
self::storeMentions($datarray['uri-id'], $text);
Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]);
self::fetchGuid($datarray);
// If we are the origin of the parent we store the original data.
@ -2715,6 +2756,7 @@ class Diaspora
$datarray["guid"] = $guid;
$datarray["uri"] = $datarray["parent-uri"] = self::getUriFromGuid($author, $guid);
$datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]);
$datarray["verb"] = Activity::POST;
$datarray["gravity"] = GRAVITY_PARENT;
@ -2722,6 +2764,8 @@ class Diaspora
$datarray["protocol"] = Conversation::PARCEL_DIASPORA;
$datarray["source"] = $xml;
/// @todo Copy tag data from original post
$prefix = share_header(
$original_item["author-name"],
$original_item["author-link"],
@ -2737,6 +2781,8 @@ class Diaspora
$datarray["body"] = $prefix.$original_item["body"]."[/share]";
Tag::storeFromBody($datarray['uri-id'], $datarray["body"]);
$datarray["tag"] = $original_item["tag"];
$datarray["attach"] = $original_item["attach"];
$datarray["app"] = $original_item["app"];
@ -2958,6 +3004,7 @@ class Diaspora
$datarray["guid"] = $guid;
$datarray["uri"] = $datarray["parent-uri"] = self::getUriFromGuid($author, $guid);
$datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]);
$datarray["verb"] = Activity::POST;
$datarray["gravity"] = GRAVITY_PARENT;
@ -2967,6 +3014,9 @@ class Diaspora
$datarray["body"] = self::replacePeopleGuid($body, $contact["url"]);
self::storeMentions($datarray['uri-id'], $text);
Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]);
if ($provider_display_name != "") {
$datarray["app"] = $provider_display_name;
}

14
src/Protocol/Feed.php
View file

@ -29,6 +29,7 @@ use Friendica\Core\Protocol;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Item;
use Friendica\Model\Tag;
use Friendica\Util\Network;
use Friendica\Util\ParseUrl;
use Friendica\Util\XML;
@ -385,6 +386,7 @@ class Feed {
}
$tags = '';
$taglist = [];
$categories = $xpath->query("category", $entry);
foreach ($categories AS $category) {
$hashtag = $category->nodeValue;
@ -394,6 +396,7 @@ class Feed {
$taglink = "#[url=" . DI::baseUrl() . "/search?tag=" . $hashtag . "]" . $hashtag . "[/url]";
$tags .= $taglink;
$taglist[] = $hashtag;
}
$body = trim(XML::getFirstNodeValue($xpath, 'atom:content/text()', $entry));
@ -475,6 +478,7 @@ class Feed {
$item["title"] = "";
$item["body"] = $item["body"] . add_page_info($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]);
$item["tag"] = add_page_keywords($item["plink"], $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]);
$taglist = get_page_keywords($item["plink"], $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]);
$item["object-type"] = Activity\ObjectType::BOOKMARK;
unset($item["attach"]);
} else {
@ -488,8 +492,11 @@ class Feed {
} else {
// @todo $preview is never set in this case, is it intended? - @MrPetovan 2018-02-13
$item["tag"] = add_page_keywords($item["plink"], $preview, true, $contact["ffi_keyword_blacklist"]);
$taglist = get_page_keywords($item["plink"], $preview, true, $contact["ffi_keyword_blacklist"]);
}
$item["body"] .= "\n" . $item['tag'];
} else {
$taglist = [];
}
// Add the link to the original feed entry if not present in feed
@ -520,6 +527,13 @@ class Feed {
$id = Item::insert($item, false, $notify);
Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id);
if (!empty($id) && !empty($taglist)) {
$feeditem = Item::selectFirst(['uri-id'], ['id' => $id]);
foreach ($taglist as $tag) {
Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
}
}
}
}

8
src/Protocol/OStatus.php
View file

@ -35,6 +35,8 @@ use Friendica\Model\Contact;
use Friendica\Model\Conversation;
use Friendica\Model\GContact;
use Friendica\Model\Item;
use Friendica\Model\ItemURI;
use Friendica\Model\Tag;
use Friendica\Model\User;
use Friendica\Network\Probe;
use Friendica\Util\DateTimeFormat;
@ -437,6 +439,7 @@ class OStatus
$item = array_merge($header, $author);
$item["uri"] = XML::getFirstNodeValue($xpath, 'atom:id/text()', $entry);
$item['uri-id'] = ItemURI::insert(['uri' => $item['uri']]);
$item["verb"] = XML::getFirstNodeValue($xpath, 'activity:verb/text()', $entry);
@ -660,6 +663,9 @@ class OStatus
}
$item['tag'] .= '#[url=' . DI::baseUrl() . '/search?tag=' . $term . ']' . $term . '[/url]';
// Store the hashtag
Tag::store($item['uri-id'], Tag::HASHTAG, $term);
}
}
}
@ -703,6 +709,8 @@ class OStatus
$item["body"] = add_page_info_to_body($item["body"]);
}
Tag::storeFromBody($item['uri-id'], $item['body']);
// Mastodon Content Warning
if (($item["verb"] == Activity::POST) && $xpath->evaluate('boolean(atom:summary)', $entry)) {
$clear_text = XML::getFirstNodeValue($xpath, 'atom:summary/text()', $entry);

7
src/Worker/TagUpdate.php
View file

@ -23,6 +23,7 @@ namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Database\DBA;
use Friendica\Model\Term;
class TagUpdate
{
@ -35,14 +36,14 @@ class TagUpdate
if ($message['uid'] == 0) {
$global = true;
DBA::update('term', ['global' => true], ['otype' => TERM_OBJ_POST, 'guid' => $message['guid']]);
DBA::update('term', ['global' => true], ['otype' => Term::OBJECT_TYPE_POST, 'guid' => $message['guid']]);
} else {
$global = (DBA::count('term', ['uid' => 0, 'otype' => TERM_OBJ_POST, 'guid' => $message['guid']]) > 0);
$global = (DBA::count('term', ['uid' => 0, 'otype' => Term::OBJECT_TYPE_POST, 'guid' => $message['guid']]) > 0);
}
$fields = ['guid' => $message['guid'], 'created' => $message['created'],
'received' => $message['received'], 'global' => $global];
DBA::update('term', $fields, ['otype' => TERM_OBJ_POST, 'oid' => $message['oid']]);
DBA::update('term', $fields, ['otype' => Term::OBJECT_TYPE_POST, 'oid' => $message['oid']]);
}
DBA::close($messages);

29
static/dbstructure.config.php
View file

@ -51,7 +51,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1338);
define('DB_UPDATE_VERSION', 1339);
}
return [
@ -1292,6 +1292,33 @@ return [
"guid" => ["guid(64)"],
]
],
"tag" => [
"comment" => "tags and mentions",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => ""],
"name" => ["type" => "varchar(96)", "not null" => "1", "default" => "", "comment" => ""],
"url" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => ""]
],
"indexes" => [
"PRIMARY" => ["id"],
"type_name_url" => ["UNIQUE", "name", "url"],
"url" => ["url"]
]
],
"post-tag" => [
"comment" => "post relation to tags",
"fields" => [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""],
"tid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["tag" => "id"], "comment" => ""],
"cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["contact" => "id"], "comment" => "Contact id of the mentioned public contact"],
],
"indexes" => [
"PRIMARY" => ["uri-id", "type", "tid", "cid"],
"uri-id" => ["tid"],
"cid" => ["tid"]
]
],
"thread" => [
"comment" => "Thread related data",
"fields" => [