Merge pull request #13778 from annando/post-counts

New table "post-counts" to precalculate the counts
This commit is contained in:
Hypolite Petovan 2023-12-28 13:13:44 -05:00 committed by GitHub
commit 5b516b1dbb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 221 additions and 4 deletions

View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2024.03-dev (Yellow Archangel)
-- DB_UPDATE_VERSION 1542
-- DB_UPDATE_VERSION 1543
-- ------------------------------------------
@ -1236,6 +1236,23 @@ CREATE TABLE IF NOT EXISTS `post-category` (
FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to categories';
--
-- TABLE post-counts
--
CREATE TABLE IF NOT EXISTS `post-counts` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`vid` smallint unsigned NOT NULL COMMENT 'Id of the verb table entry that contains the activity verbs',
`reaction` varchar(1) NOT NULL COMMENT 'Emoji Reaction',
`parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri',
`count` int unsigned DEFAULT 0 COMMENT 'Number of activities',
PRIMARY KEY(`uri-id`,`vid`,`reaction`),
INDEX `vid` (`vid`),
INDEX `parent-uri-id` (`parent-uri-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT,
FOREIGN KEY (`parent-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Original remote activity';
--
-- TABLE post-collection
--

View file

@ -61,6 +61,7 @@ Database Tables
| [post-category](help/database/db_post-category) | post relation to categories |
| [post-collection](help/database/db_post-collection) | Collection of posts |
| [post-content](help/database/db_post-content) | Content for all posts |
| [post-counts](help/database/db_post-counts) | Original remote activity |
| [post-delivery](help/database/db_post-delivery) | Delivery data for posts for the batch processing |
| [post-delivery-data](help/database/db_post-delivery-data) | Delivery data for items |
| [post-engagement](help/database/db_post-engagement) | Engagement data per post |

View file

@ -0,0 +1,35 @@
Table post-counts
===========
Original remote activity
Fields
------
| Field | Description | Type | Null | Key | Default | Extra |
| ------------- | ----------------------------------------------------------- | ----------------- | ---- | --- | ------- | ----- |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | |
| vid | Id of the verb table entry that contains the activity verbs | smallint unsigned | NO | PRI | NULL | |
| reaction | Emoji Reaction | varchar(1) | NO | PRI | NULL | |
| parent-uri-id | Id of the item-uri table that contains the parent uri | int unsigned | YES | | NULL | |
| count | Number of activities | int unsigned | YES | | 0 | |
Indexes
------------
| Name | Fields |
| ------------- | --------------------- |
| PRIMARY | uri-id, vid, reaction |
| vid | vid |
| parent-uri-id | parent-uri-id |
Foreign Keys
------------
| Field | Target Table | Target Field |
|-------|--------------|--------------|
| uri-id | [item-uri](help/database/db_item-uri) | id |
| vid | [verb](help/database/db_verb) | id |
| parent-uri-id | [item-uri](help/database/db_item-uri) | id |
Return to [database documentation](help/database)

View file

@ -32,6 +32,7 @@ use Friendica\Model\ItemURI;
use Friendica\Model\Photo;
use Friendica\Model\Post;
use Friendica\Model\Post\Category;
use Friendica\Model\Post\Counts;
use Friendica\Model\Tag;
use Friendica\Model\Verb;
use Friendica\Protocol\ActivityPub\Processor;
@ -51,7 +52,7 @@ class PostUpdate
// Needed for the helper function to read from the legacy term table
const OBJECT_TYPE_POST = 1;
const VERSION = 1507;
const VERSION = 1543;
/**
* Calls the post update functions
@ -124,6 +125,9 @@ class PostUpdate
if (!self::update1507()) {
return false;
}
if (!self::update1543()) {
return false;
}
return true;
}
@ -1303,4 +1307,51 @@ class PostUpdate
return false;
}
/**
* Create "post-counts" entries for old entries.
*
* @return bool "true" when the job is done
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
*/
private static function update1543()
{
// Was the script completed?
if (DI::keyValue()->get('post_update_version') >= 1543) {
return true;
}
$id = DI::keyValue()->get('post_update_version_1543_id') ?? 0;
Logger::info('Start', ['uri-id' => $id]);
$rows = 0;
$posts = Post::selectPosts(['uri-id', 'parent-uri-id'], ["`uri-id` > ? AND `gravity` IN (?, ?)", $id, Item::GRAVITY_COMMENT, Item::GRAVITY_PARENT], ['order' => ['uri-id'], 'limit' => 1000]);
if (DBA::errorNo() != 0) {
Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
return false;
}
while ($post = Post::fetch($posts)) {
$id = $post['uri-id'];
Counts::updateForPost($post['uri-id'], $post['parent-uri-id']);
++$rows;
}
DBA::close($posts);
DI::keyValue()->set('post_update_version_1543_id', $id);
Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
if ($rows <= 100) {
DI::keyValue()->set('post_update_version', 1543);
Logger::info('Done');
return true;
}
return false;
}
}

View file

@ -1432,6 +1432,10 @@ class Item
self::updateDisplayCache($posted_item['uri-id']);
}
if (in_array($posted_item['gravity'], [self::GRAVITY_ACTIVITY, self::GRAVITY_COMMENT]) && ($posted_item['uid'] == 0)) {
Post\Counts::update($posted_item['thr-parent-id'], $posted_item['parent-uri-id'], $posted_item['vid'], $posted_item['verb'], $posted_item['body']);
}
if ($inserted) {
Post\Engagement::storeFromItem($posted_item);
}

View file

@ -72,7 +72,7 @@ class Post
if (array_key_exists('title', $row)) {
$row['title'] = '';
}
if (array_key_exists('body', $row)) {
if (array_key_exists('body', $row) && empty($row['body'])) {
$row['body'] = $row['verb'];
}
if (array_key_exists('object', $row)) {

89
src/Model/Post/Counts.php Normal file
View file

@ -0,0 +1,89 @@
<?php
/**
* @copyright Copyright (C) 2010-2023, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Model\Post;
use Friendica\Content\Smilies;
use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\Model\Item;
use Friendica\Model\Post;
use Friendica\Model\Verb;
use Friendica\Protocol\Activity;
class Counts
{
/**
* Insert or update a post-counts entry
*
* @param int $uri_id
*/
public static function update(int $uri_id, int $parent_uri_id, int $vid, string $verb, string $body = null)
{
$condition = ['thr-parent-id' => $uri_id, 'vid' => $vid];
if ($body == $verb) {
$condition['body'] = null;
$body = '';
} elseif (($verb != Activity::POST) && (mb_strlen($body) == 1) && Smilies::isEmojiPost($body)) {
$condition['body'] = $body;
} else {
$body = '';
}
$fields = [
'uri-id' => $uri_id,
'vid' => $vid,
'reaction' => $body,
'parent-uri-id' => $parent_uri_id,
'count' => Post::countPosts($condition),
];
if ($fields['count'] == 0) {
return true;
}
return DBA::insert('post-counts', $fields, Database::INSERT_UPDATE);
}
public static function updateForPost(int $uri_id, int $parent_uri_id)
{
self::update($uri_id, $parent_uri_id, Verb::getID(Activity::POST), Activity::POST);
$activities = DBA::p("SELECT `parent-uri-id`, `vid`, `verb`, `body` FROM `post-view` WHERE `thr-parent-id` = ? AND `gravity` = ? GROUP BY `parent-uri-id`, `vid`, `verb`, `body`", $uri_id, Item::GRAVITY_ACTIVITY);
while ($activity = DBA::fetch($activities)) {
self::update($uri_id, $activity['parent-uri-id'], $activity['vid'], $activity['verb'], $activity['body']);
}
DBA::close($activities);
}
/**
* Retrieves counts of the given uri-id
*
* @param int $uriId
*
* @return array
*/
public static function getByURIId(int $uriId): array
{
return DBA::selectToArray('post-counts', [], ['uri-id' => $uriId]);
}
}

View file

@ -1272,6 +1272,11 @@ class Receiver
}
}
if (empty($receivers) && !empty($parent['parent-author-link'])) {
$uid = User::getIdForURL($parent['parent-author-link']);
$receivers[$uid] = ['uid' => $uid, 'type' => self::TARGET_BTO];
}
if (!empty($reply) && (!empty($receivers[0]) || !empty($receivers[-1]))) {
$parents = Post::select(['uid'], DBA::mergeConditions(['uri' => $reply], ["`uid` != ?", 0]));
while ($parent = Post::fetch($parents)) {

View file

@ -56,7 +56,7 @@ use Friendica\Database\DBA;
// This file is required several times during the test in DbaDefinition which justifies this condition
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1542);
define('DB_UPDATE_VERSION', 1543);
}
return [
@ -1264,6 +1264,21 @@ return [
"uid_uri-id" => ["uid", "uri-id"],
]
],
"post-counts" => [
"comment" => "Original remote activity",
"fields" => [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"vid" => ["type" => "smallint unsigned", "not null" => "1", "primary" => "1", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"],
"reaction" => ["type" => "varchar(1)", "not null" => "1", "primary" => "1", "comment" => "Emoji Reaction"],
"parent-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table that contains the parent uri"],
"count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of activities"],
],
"indexes" => [
"PRIMARY" => ["uri-id", "vid", "reaction"],
"vid" => ["vid"],
"parent-uri-id" => ["parent-uri-id"],
]
],
"post-collection" => [
"comment" => "Collection of posts",
"fields" => [