strk/friendica
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

File and category aren't using "term" anymore

Browse source
This commit is contained in:
Michael 2020-05-03 15:13:40 +00:00
parent 233d6eea2b
commit 875d6b36e9
10 changed files with 276 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
mod/network.php
View file

@ -37,8 +37,8 @@ use Friendica\DI;
use Friendica\Model\Contact;
use Friendica\Model\Group;
use Friendica\Model\Item;
use Friendica\Model\Post\Category;
use Friendica\Model\Profile;
use Friendica\Model\Term;
use Friendica\Module\Security\Login;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Proxy as ProxyUtils;
@ -379,25 +379,25 @@ function networkFlatView(App $a, $update = 0)
networkPager($a, $pager, $update);
$item_params = ['order' => ['id' => true]];
if (strlen($file)) {
$term_condition = ["`term` = ? AND `otype` = ? AND `type` = ? AND `uid` = ?",
$file, Term::OBJECT_TYPE_POST, Term::FILE, local_user()];
$item_params = ['order' => ['uri-id' => true]];
$term_condition = ['name' => $file, 'type' => Category::FILE, 'uid' => local_user()];
$term_params = ['order' => ['tid' => true], 'limit' => [$pager->getStart(), $pager->getItemsPerPage()]];
$result = DBA::select('term', ['oid'], $term_condition, $term_params);
$result = DBA::select('category-view', ['uri-id'], $term_condition, $term_params);
$posts = [];
while ($term = DBA::fetch($result)) {
$posts[] = $term['oid'];
$posts[] = $term['uri-id'];
}
DBA::close($result);
if (count($posts) == 0) {
return '';
}
$item_condition = ['uid' => local_user(), 'id' => $posts];
$item_condition = ['uid' => local_user(), 'uri-id' => $posts];
} else {
$item_params = ['order' => ['id' => true]];
$item_condition = ['uid' => local_user()];
$item_params['limit'] = [$pager->getStart(), $pager->getItemsPerPage()];

67
src/Database/PostUpdate.php
View file

@ -28,7 +28,9 @@ use Friendica\Model\Contact;
use Friendica\Model\Item;
use Friendica\Model\ItemURI;
use Friendica\Model\PermissionSet;
use Friendica\Model\Post\Category;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Model\UserItem;
use Friendica\Util\Strings;
@ -72,6 +74,9 @@ class PostUpdate
if (!self::update1342()) {
return false;
}
if (!self::update1346()) {
return false;
}
return true;
}
@ -667,4 +672,66 @@ class PostUpdate
return false;
}
/**
* Fill the "tag" table with tags and mentions from the "term" table
*
* @return bool "true" when the job is done
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
private static function update1346()
{
// Was the script completed?
if (DI::config()->get('system', 'post_update_version') >= 1346) {
return true;
}
$id = DI::config()->get('system', 'post_update_version_1346_id', 0);
Logger::info('Start', ['item' => $id]);
$rows = 0;
$terms = DBA::select('term', ['oid'],
["`type` IN (?, ?) AND `oid` >= ?", Category::CATEGORY, Category::FILE, $id],
['order' => ['oid'], 'limit' => 1000, 'group_by' => ['oid']]);
if (DBA::errorNo() != 0) {
Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
return false;
}
while ($term = DBA::fetch($terms)) {
$item = Item::selectFirst(['uri-id', 'uid'], ['id' => $term['oid']]);
if (!DBA::isResult($item)) {
continue;
}
$file = Term::fileTextFromItemId($term['oid']);
if (!empty($file)) {
Category::storeTextByURIId($item['uri-id'], $item['uid'], $file);
}
$id = $term['oid'];
++$rows;
if ($rows % 100 == 0) {
DI::config()->set('system', 'post_update_version_1346_id', $id);
}
}
DBA::close($terms);
DI::config()->set('system', 'post_update_version_1346_id', $id);
Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
// When there are less than 10 items processed this means that we reached the end
// The other entries will then be processed with the regular functionality
if ($rows < 10) {
DI::config()->set('system', 'post_update_version', 1346);
Logger::info('Done');
return true;
}
return false;
}
}

27
src/Model/FileTag.php
View file

@ -23,7 +23,7 @@ namespace Friendica\Model;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Term;
use Friendica\Model\Post\Category;
/**
* This class handles FileTag related functions
@ -196,11 +196,11 @@ class FileTag
if ($type == 'file') {
$lbracket = '[';
$rbracket = ']';
$termtype = Term::FILE;
$termtype = Category::FILE;
} else {
$lbracket = '<';
$rbracket = '>';
$termtype = Term::CATEGORY;
$termtype = Category::CATEGORY;
}
$filetags_updated = $saved;
@ -224,13 +224,7 @@ class FileTag
}
foreach ($deleted_tags as $key => $tag) {
$r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d",
DBA::escape($tag),
intval(Term::OBJECT_TYPE_POST),
intval($termtype),
intval($uid));
if (DBA::isResult($r)) {
if (DBA::exists('category-view', ['name' => $tag, 'type' => $termtype, 'uid' => $uid])) {
unset($deleted_tags[$key]);
} else {
$filetags_updated = str_replace($lbracket . self::encode($tag) . $rbracket, '', $filetags_updated);
@ -303,10 +297,10 @@ class FileTag
if ($cat == true) {
$pattern = '<' . self::encode($file) . '>';
$termtype = Term::CATEGORY;
$termtype = Category::CATEGORY;
} else {
$pattern = '[' . self::encode($file) . ']';
$termtype = Term::FILE;
$termtype = Category::FILE;
}
$item = Item::selectFirst(['file'], ['id' => $item_id, 'uid' => $uid]);
@ -319,14 +313,7 @@ class FileTag
Item::update($fields, ['id' => $item_id]);
$r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d",
DBA::escape($file),
intval(Term::OBJECT_TYPE_POST),
intval($termtype),
intval($uid)
);
if (!DBA::isResult($r)) {
if (!DBA::exists('category-view', ['name' => $file, 'type' => $termtype, 'uid' => $uid])) {
$saved = DI::pConfig()->get($uid, 'system', 'filetags');
DI::pConfig()->set($uid, 'system', 'filetags', str_replace($pattern, '', $saved));
}

21
src/Model/Item.php
View file

@ -32,6 +32,7 @@ use Friendica\Core\System;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Post\Category;
use Friendica\Protocol\Activity;
use Friendica\Protocol\ActivityPub;
use Friendica\Protocol\Diaspora;
@ -319,7 +320,7 @@ class Item
if (!array_key_exists('verb', $row) || in_array($row['verb'], ['', Activity::POST, Activity::SHARE])) {
// Build the file string out of the term entries
if (array_key_exists('file', $row) && empty($row['file'])) {
$row['file'] = Term::fileTextFromItemId($row['internal-iid']);
$row['file'] = Category::getTextByURIId($row['internal-uri-id'], $row['internal-uid']);
}
}
@ -345,7 +346,8 @@ class Item
// Remove internal fields
unset($row['internal-activity']);
unset($row['internal-network']);
unset($row['internal-iid']);
unset($row['internal-uri-id']);
unset($row['internal-uid']);
unset($row['internal-psid']);
unset($row['internal-iaid']);
unset($row['internal-user-ignored']);
@ -671,7 +673,8 @@ class Item
'resource-id', 'event-id', 'attach', 'post-type', 'file',
'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark',
'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global',
'id' => 'item_id', 'network', 'icid', 'iaid', 'id' => 'internal-iid',
'id' => 'item_id', 'network', 'icid', 'iaid',
'uri-id' => 'internal-uri-id', 'uid' => 'internal-uid',
'network' => 'internal-network', 'iaid' => 'internal-iaid', 'psid' => 'internal-psid'];
if ($usermode) {
@ -834,7 +837,7 @@ class Item
private static function constructSelectFields(array $fields, array $selected)
{
if (!empty($selected)) {
$selected = array_merge($selected, ['internal-iid', 'internal-psid', 'internal-iaid', 'internal-network']);
$selected = array_merge($selected, ['internal-uri-id', 'internal-uid', 'internal-psid', 'internal-iaid', 'internal-network']);
}
if (in_array('verb', $selected)) {
@ -917,7 +920,7 @@ class Item
// We cannot simply expand the condition to check for origin entries
// The condition needn't to be a simple array but could be a complex condition.
// And we have to execute this query before the update to ensure to fetch the same data.
$items = DBA::select('item', ['id', 'origin', 'uri', 'uri-id', 'iaid', 'icid', 'file'], $condition);
$items = DBA::select('item', ['id', 'origin', 'uri', 'uri-id', 'iaid', 'icid', 'uid', 'file'], $condition);
$content_fields = [];
foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
@ -1013,7 +1016,7 @@ class Item
}
if (!is_null($files)) {
Term::insertFromFileFieldByItemId($item['id'], $files);
Category::storeTextByURIId($item['uri-id'], $item['uid'], $files);
if (!empty($item['file'])) {
DBA::update('item', ['file' => ''], ['id' => $item['id']]);
}
@ -1099,7 +1102,7 @@ class Item
{
Logger::info('Mark item for deletion by id', ['id' => $item_id, 'callstack' => System::callstack()]);
// locate item to be deleted
$fields = ['id', 'uri', 'uid', 'parent', 'parent-uri', 'origin',
$fields = ['id', 'uri', 'uri-id', 'uid', 'parent', 'parent-uri', 'origin',
'deleted', 'file', 'resource-id', 'event-id', 'attach',
'verb', 'object-type', 'object', 'target', 'contact-id',
'icid', 'iaid', 'psid'];
@ -1172,7 +1175,7 @@ class Item
$item_fields = ['deleted' => true, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()];
DBA::update('item', $item_fields, ['id' => $item['id']]);
Term::insertFromFileFieldByItemId($item['id'], '');
Category::storeTextByURIId($item['uri-id'], $item['uid'], '');
self::deleteThread($item['id'], $item['parent-uri']);
if (!self::exists(["`uri` = ? AND `uid` != 0 AND NOT `deleted`", $item['uri']])) {
@ -1949,7 +1952,7 @@ class Item
* This is not perfect - but a workable solution until we found the reason for the problem.
*/
if (!empty($files)) {
Term::insertFromFileFieldByItemId($current_post, $files);
Category::storeTextByURIId($item['uri-id'], $item['uid'], $files);
}
// In that function we check if this is a forum post. Additionally we delete the item under certain circumstances

119
src/Model/Post/Category.php Normal file
View file

@ -0,0 +1,119 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Model\Post;
use Friendica\Database\DBA;
use Friendica\Model\Item;
use Friendica\Model\Tag;
/**
* Class Category
*
* This Model class handles category table interactions.
* This tables stores user-applied categories related to posts.
*/
class Category
{
const UNKNOWN = 0;
const CATEGORY = 3;
const FILE = 5;
/**
* Generates the legacy item.file field string from an item ID.
* Includes only file and category terms.
*
* @param int $item_id
* @return string
* @throws \Exception
*/
public static function getTextByURIId(int $uri_id, int $uid)
{
$file_text = '';
$tags = DBA::selectToArray('post-category', ['type', 'name'], ['uri-id' => $uri_id, 'uid' => $uid]);
foreach ($tags as $tag) {
if ($tag['type'] == self::CATEGORY) {
$file_text .= '<' . $tag['term'] . '>';
} else {
$file_text .= '[' . $tag['term'] . ']';
}
}
return $file_text;
}
/**
* Inserts new terms for the provided item ID based on the legacy item.file field BBCode content.
* Deletes all previous file terms for the same item ID.
*
* @param integer $item_id item id
* @param $files
* @return void
* @throws \Exception
*/
public static function storeTextByURIId(int $uri_id, int $uid, string $files)
{
$message = Item::selectFirst(['deleted'], ['uri-id' => $uri_id, 'uid' => $uid]);
if (!DBA::isResult($message)) {
return;
}
// Clean up all tags
DBA::delete('post-category', ['uri-id' => $uri_id, 'uid' => $uid]);
if ($message['deleted']) {
return;
}
if (preg_match_all("/\[(.*?)\]/ism", $files, $result)) {
foreach ($result[1] as $file) {
$tagid = Tag::getID($file);
if (empty($tagid)) {
continue;
}
DBA::insert('post-category', [
'uri-id' => $uri_id,
'uid' => $uid,
'type' => self::FILE,
'tid' => $tagid
]);
}
}
if (preg_match_all("/\<(.*?)\>/ism", $files, $result)) {
foreach ($result[1] as $file) {
$tagid = Tag::getID($file);
if (empty($tagid)) {
continue;
}
DBA::insert('post-category', [
'uri-id' => $uri_id,
'uid' => $uid,
'type' => self::CATEGORY,
'tid' => $tagid
]);
}
}
}
}

40
src/Model/Tag.php
View file

@ -124,22 +124,14 @@ class Tag
}
if (empty($cid)) {
$fields = ['name' => substr($name, 0, 96), 'url' => ''];
if (($type != self::HASHTAG) && !empty($url) && ($url != $name)) {
$fields['url'] = strtolower($url);
}
$tag = DBA::selectFirst('tag', ['id'], $fields);
if (!DBA::isResult($tag)) {
DBA::insert('tag', $fields, true);
$tagid = DBA::lastInsertId();
$url = strtolower($url);
} else {
$tagid = $tag['id'];
$url = '';
}
$tagid = self::getID($name, $url);
if (empty($tagid)) {
Logger::error('No tag id created', $fields);
return;
}
}
@ -160,6 +152,32 @@ class Tag
Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'name' => $name, 'type' => $type, 'callstack' => System::callstack(8)]);
}
/**
* Get a tag id for a given tag name and url
*
* @param string $name
* @param string $url
* @return void
*/
public static function getID(string $name, string $url = '')
{
$fields = ['name' => substr($name, 0, 96), 'url' => $url];
$tag = DBA::selectFirst('tag', ['id'], $fields);
if (DBA::isResult($tag)) {
return $tag['id'];
}
DBA::insert('tag', $fields, true);
$tid = DBA::lastInsertId();
if (!empty($tid)) {
return $tid;
}
Logger::error('No tag id created', $fields);
return 0;
}
/**
* Store tag/mention elements
*

6
src/Module/Profile/Status.php
View file

@ -29,9 +29,9 @@ use Friendica\Core\Session;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Item;
use Friendica\Model\Post\Category;
use Friendica\Model\Profile as ProfileModel;
use Friendica\Model\User;
use Friendica\Model\Term;
use Friendica\Module\BaseProfile;
use Friendica\Module\Security\Login;
use Friendica\Util\DateTimeFormat;
@ -142,8 +142,8 @@ class Status extends BaseProfile
$sql_post_table = "";
if (!empty($category)) {
$sql_post_table = sprintf("INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ",
DBA::escape(Strings::protectSprintf($category)), intval(Term::OBJECT_TYPE_POST), intval(Term::CATEGORY), intval($a->profile['uid']));
$sql_post_table = sprintf("INNER JOIN (SELECT `uri-id` FROM `category-view` WHERE `name` = '%s' AND `type` = %d AND `uid` = %d ORDER BY `uri-id` DESC) AS `category` ON `item`.`uri-id` = `category`.`uri-id` ",
DBA::escape(Strings::protectSprintf($category)), intval(Category::CATEGORY), intval($a->profile['uid']));
}
if (!empty($hashtags)) {

11
src/Protocol/DFRN.php
View file

@ -39,9 +39,9 @@ use Friendica\Model\ItemURI;
use Friendica\Model\Mail;
use Friendica\Model\Notify\Type;
use Friendica\Model\PermissionSet;
use Friendica\Model\Post\Category;
use Friendica\Model\Profile;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Model\User;
use Friendica\Network\Probe;
use Friendica\Util\Crypto;
@ -241,13 +241,8 @@ class DFRN
}
if (isset($category)) {
$sql_post_table = sprintf(
"INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ",
DBA::escape(Strings::protectSprintf($category)),
intval(Term::OBJECT_TYPE_POST),
intval(Term::CATEGORY),
intval($owner_id)
);
$sql_post_table = sprintf("INNER JOIN (SELECT `uri-id` FROM `category-view` WHERE `name` = '%s' AND `type` = %d AND `uid` = %d ORDER BY `uri-id` DESC) AS `category` ON `item`.`uri-id` = `category`.`uri-id` ",
DBA::escape(Strings::protectSprintf($category)), intval(Category::CATEGORY), intval($owner_id));
}
if ($public_feed && ! $converse) {

15
static/dbstructure.config.php
View file

@ -51,7 +51,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1343);
define('DB_UPDATE_VERSION', 1345);
}
return [
@ -1306,6 +1306,19 @@ return [
"url" => ["url"]
]
],
"post-category" => [
"comment" => "post relation to categories",
"fields" => [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "User id"],
"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""],
"tid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["tag" => "id"], "comment" => ""],
],
"indexes" => [
"PRIMARY" => ["uri-id", "uid", "type", "tid"],
"uri-id" => ["tid"]
]
],
"post-tag" => [
"comment" => "post relation to tags",
"fields" => [

15
static/dbview.config.php
View file

@ -37,6 +37,21 @@
*/
return [
"category-view" => [
"fields" => [
"uri-id" => ["post-category", "uri-id"],
"uid" => ["post-category", "uid"],
"uri" => ["item-uri", "uri"],
"guid" => ["item-uri", "guid"],
"type" => ["post-category", "type"],
"tid" => ["post-category", "tid"],
"name" => ["tag", "name"],
"url" => ["tag", "url"],
],
"query" => "FROM `post-category`
INNER JOIN `item-uri` ON `item-uri`.id = `post-category`.`uri-id`
LEFT JOIN `tag` ON `post-category`.`tid` = `tag`.`id`"
],
"tag-view" => [
"fields" => [
"uri-id" => ["post-tag", "uri-id"],