Browse Source

Several performance improvements

pull/11845/head
Michael 2 months ago
parent
commit
5b3145d7ce
  1. 14
      database.sql
  2. 2
      doc/database/db_contact.md
  3. 1
      doc/database/db_notification.md
  4. 19
      doc/database/db_post-collection.md
  5. 36
      src/Model/APContact.php
  6. 2
      src/Model/FContact.php
  7. 12
      src/Model/Post.php
  8. 5
      src/Model/Post/Collection.php
  9. 4
      src/Module/Api/Mastodon/Statuses/Pin.php
  10. 4
      src/Module/Item/Pin.php
  11. 26
      src/Protocol/ActivityPub/Processor.php
  12. 23
      src/Protocol/ActivityPub/Transmitter.php
  13. 7
      static/dbstructure.config.php
  14. 4
      static/dbview.config.php
  15. 10
      update.php

14
database.sql

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2022.09-dev (Giant Rhubarb)
-- DB_UPDATE_VERSION 1480
-- DB_UPDATE_VERSION 1481
-- ------------------------------------------
@ -222,7 +222,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
INDEX `uid_contact-type` (`uid`,`contact-type`),
INDEX `uid_self_contact-type` (`uid`,`self`,`contact-type`),
INDEX `self_network_uid` (`self`,`network`,`uid`),
INDEX `gsid` (`gsid`),
INDEX `gsid_uid_failed` (`gsid`,`uid`,`failed`),
INDEX `uri-id` (`uri-id`),
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
@ -920,6 +920,7 @@ CREATE TABLE IF NOT EXISTS `notification` (
INDEX `target-uri-id` (`target-uri-id`),
INDEX `parent-uri-id` (`parent-uri-id`),
INDEX `seen_uid` (`seen`,`uid`),
INDEX `uid_type_parent-uri-id_actor-id` (`uid`,`type`,`parent-uri-id`,`actor-id`),
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT,
FOREIGN KEY (`actor-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
@ -1153,9 +1154,12 @@ CREATE TABLE IF NOT EXISTS `post-category` (
CREATE TABLE IF NOT EXISTS `post-collection` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0 - Featured',
`author-id` int unsigned COMMENT 'Author of the featured post',
PRIMARY KEY(`uri-id`,`type`),
INDEX `type` (`type`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
INDEX `author-id` (`author-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Collection of posts';
--
@ -2423,7 +2427,7 @@ DROP VIEW IF EXISTS `collection-view`;
CREATE VIEW `collection-view` AS SELECT
`post-collection`.`uri-id` AS `uri-id`,
`post-collection`.`type` AS `type`,
`post`.`author-id` AS `cid`,
`post-collection`.`author-id` AS `cid`,
`post`.`received` AS `received`,
`post`.`created` AS `created`,
`post-thread`.`commented` AS `commented`,
@ -2431,7 +2435,7 @@ CREATE VIEW `collection-view` AS SELECT
`post`.`visible` AS `visible`,
`post`.`deleted` AS `deleted`,
`post`.`thr-parent-id` AS `thr-parent-id`,
`post`.`author-id` AS `author-id`,
`post-collection`.`author-id` AS `author-id`,
`post`.`gravity` AS `gravity`
FROM `post-collection`
INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`

2
doc/database/db_contact.md

@ -121,7 +121,7 @@ Indexes
| uid_contact-type | uid, contact-type |
| uid_self_contact-type | uid, self, contact-type |
| self_network_uid | self, network, uid |
| gsid | gsid |
| gsid_uid_failed | gsid, uid, failed |
| uri-id | uri-id |
Foreign Keys

1
doc/database/db_notification.md

@ -31,6 +31,7 @@ Indexes
| target-uri-id | target-uri-id |
| parent-uri-id | parent-uri-id |
| seen_uid | seen, uid |
| uid_type_parent-uri-id_actor-id | uid, type, parent-uri-id, actor-id |
Foreign Keys
------------

19
doc/database/db_post-collection.md

@ -6,18 +6,20 @@ Collection of posts
Fields
------
| Field | Description | Type | Null | Key | Default | Extra |
| ------ | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | |
| type | 0 - Featured | tinyint unsigned | NO | PRI | 0 | |
| Field | Description | Type | Null | Key | Default | Extra |
| --------- | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | |
| type | 0 - Featured | tinyint unsigned | NO | PRI | 0 | |
| author-id | Author of the featured post | int unsigned | YES | | NULL | |
Indexes
------------
| Name | Fields |
| ------- | ------------ |
| PRIMARY | uri-id, type |
| type | type |
| Name | Fields |
| --------- | ------------ |
| PRIMARY | uri-id, type |
| type | type |
| author-id | author-id |
Foreign Keys
------------
@ -25,5 +27,6 @@ Foreign Keys
| Field | Target Table | Target Field |
|-------|--------------|--------------|
| uri-id | [item-uri](help/database/db_item-uri) | id |
| author-id | [contact](help/database/db_contact) | id |
Return to [database documentation](help/database)

36
src/Model/APContact.php

@ -24,6 +24,7 @@ namespace Friendica\Model;
use Friendica\Content\Text\HTML;
use Friendica\Core\Cache\Enum\Duration;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\System;
use Friendica\Database\DBA;
use Friendica\DI;
@ -352,17 +353,18 @@ class APContact
if (!empty($apcontact['outbox'])) {
if (!empty($local_owner)) {
$outbox = ActivityPub\Transmitter::getOutbox($local_owner);
$statuses_count = self::getStatusesCount($local_owner);
} else {
$outbox = ActivityPub::fetchContent($apcontact['outbox']);
$statuses_count = $outbox['totalItems'] ?? 0;
}
if (!empty($outbox['totalItems'])) {
if (!empty($statuses_count)) {
// Mastodon seriously allows for this condition?
// Jul 20 2021 - See https://chaos.social/@m11 for a negative posts count
if ($outbox['totalItems'] < 0) {
$outbox['totalItems'] = 0;
if ($statuses_count < 0) {
$statuses_count = 0;
}
$apcontact['statuses_count'] = $outbox['totalItems'];
$apcontact['statuses_count'] = $statuses_count;
}
}
@ -483,6 +485,30 @@ class APContact
return DBA::selectFirst('apcontact', [], ['url' => $apcontact['url']]) ?: [];
}
/**
* Fetch the number of statuses for the given owner
*
* @param array $owner
*
* @return integer
*/
private static function getStatusesCount(array $owner): int
{
$condition = [
'private' => [Item::PUBLIC, Item::UNLISTED],
'author-id' => Contact::getIdForURL($owner['url'], 0, false),
'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT],
'network' => Protocol::DFRN,
'parent-network' => Protocol::FEDERATED,
'deleted' => false,
'visible' => true
];
$count = Post::countPosts($condition);
return $count;
}
/**
* Mark the given AP Contact as "to archive"
*

2
src/Model/FContact.php

@ -100,7 +100,7 @@ class FContact
$interacted = DBA::count('contact-relation', ["`cid` = ? AND NOT `follows` AND `last-interaction` > ?", $contact['id'], $last_interaction]);
$interacting = DBA::count('contact-relation', ["`relation-cid` = ? AND NOT `follows` AND `last-interaction` > ?", $contact['id'], $last_interaction]);
$posts = Post::countPosts(['author-id' => $contact['id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]]);
$posts = DBA::count('post', ['author-id' => $contact['id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]]);
}
$fields = [

12
src/Model/Post.php

@ -400,12 +400,12 @@ class Post
AND (NOT `causer-blocked` OR `causer-id` = ? OR `causer-id` IS NULL) AND NOT `contact-blocked`
AND ((NOT `contact-readonly` AND NOT `contact-pending` AND (`contact-rel` IN (?, ?)))
OR `self` OR `gravity` != ? OR `contact-uid` = ?)
AND NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `uid` = ? AND `uri-id` = `" . $view . "`.`uri-id` AND `hidden`)
AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `author-id` AND `blocked`)
AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `owner-id` AND `blocked`)
AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `author-id` AND `ignored` AND `gravity` = ?)
AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `owner-id` AND `ignored` AND `gravity` = ?)",
0, Contact::SHARING, Contact::FRIEND, GRAVITY_PARENT, 0, $uid, $uid, $uid, $uid, GRAVITY_PARENT, $uid, GRAVITY_PARENT]);
AND NOT `" . $view . "`.`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ? AND `hidden`)
AND NOT `author-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `blocked`)
AND NOT `owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `blocked`)
AND NOT (`gravity` = ? AND `author-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `ignored`))
AND NOT (`gravity` = ? AND `owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `ignored`))",
0, Contact::SHARING, Contact::FRIEND, GRAVITY_PARENT, 0, $uid, $uid, $uid, GRAVITY_PARENT, $uid, GRAVITY_PARENT, $uid]);
$select_string = implode(', ', array_map([DBA::class, 'quoteIdentifier'], $selected));

5
src/Model/Post/Collection.php

@ -37,15 +37,16 @@ class Collection
*
* @param integer $uri_id
* @param integer $type
* @param integer $author_id
* @param integer $cache_uid If set to a non zero value, the featured cache is cleared
*/
public static function add(int $uri_id, int $type, int $cache_uid = 0)
public static function add(int $uri_id, int $type, int $author_id, int $cache_uid = 0)
{
if (empty($uri_id)) {
throw new BadMethodCallException('Empty URI_id');
}
DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type], Database::INSERT_IGNORE);
DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type, 'author-id' => $author_id], Database::INSERT_IGNORE);
if (!empty($cache_uid) && ($type == self::FEATURED)) {
DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_FEATURED . $cache_uid);

4
src/Module/Api/Mastodon/Statuses/Pin.php

@ -41,12 +41,12 @@ class Pin extends BaseApi
DI::mstdnError()->UnprocessableEntity();
}
$item = Post::selectFirstForUser($uid, ['id', 'gravity'], ['uri-id' => $this->parameters['id'], 'uid' => [$uid, 0]]);
$item = Post::selectFirstForUser($uid, ['id', 'gravity', 'author-id'], ['uri-id' => $this->parameters['id'], 'uid' => [$uid, 0]]);
if (!DBA::isResult($item)) {
DI::mstdnError()->RecordNotFound();
}
Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $uid);
Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $item['author-id'], $uid);
System::jsonExit(DI::mstdnStatus()->createFromUriId($this->parameters['id'], $uid)->toArray());
}

4
src/Module/Item/Pin.php

@ -48,7 +48,7 @@ class Pin extends BaseModule
$itemId = intval($this->parameters['id']);
$item = Post::selectFirst(['uri-id', 'uid', 'featured'], ['id' => $itemId]);
$item = Post::selectFirst(['uri-id', 'uid', 'featured', 'author-id'], ['id' => $itemId]);
if (!DBA::isResult($item)) {
throw new HTTPException\NotFoundException();
}
@ -60,7 +60,7 @@ class Pin extends BaseModule
$pinned = !$item['featured'];
if ($pinned) {
Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, local_user());
Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id'], local_user());
} else {
Post\Collection::remove($item['uri-id'], Post\Collection::FEATURED, local_user());
}

26
src/Protocol/ActivityPub/Processor.php

@ -679,7 +679,7 @@ class Processor
* Fetch the Uri-Id of a post for the "featured" collection
*
* @param array $activity
* @return null|int
* @return null|array
*/
private static function getUriIdForFeaturedCollection(array $activity)
{
@ -697,7 +697,7 @@ class Processor
}
}
$parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]);
$parent = Post::selectFirst(['uri-id', 'author-id'], ['uri' => $activity['object_id']]);
if (empty($parent['uri-id'])) {
if (self::fetchMissingActivity($activity['object_id'], $activity, '', Receiver::COMPLETION_AUTO)) {
$parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]);
@ -705,7 +705,7 @@ class Processor
}
if (!empty($parent['uri-id'])) {
return $parent['uri-id'];
$parent;
}
return null;
@ -718,14 +718,14 @@ class Processor
*/
public static function addToFeaturedCollection(array $activity)
{
$uriid = self::getUriIdForFeaturedCollection($activity);
if (empty($uriid)) {
$post = self::getUriIdForFeaturedCollection($activity);
if (empty($post)) {
return;
}
Logger::debug('Add post to featured collection', ['uri-id' => $uriid]);
Logger::debug('Add post to featured collection', ['post' => $post]);
Post\Collection::add($uriid, Post\Collection::FEATURED);
Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']);
Queue::remove($activity);
}
@ -736,14 +736,14 @@ class Processor
*/
public static function removeFromFeaturedCollection(array $activity)
{
$uriid = self::getUriIdForFeaturedCollection($activity);
if (empty($uriid)) {
$post = self::getUriIdForFeaturedCollection($activity);
if (empty($post)) {
return;
}
Logger::debug('Remove post from featured collection', ['uri-id' => $uriid]);
Logger::debug('Remove post from featured collection', ['post' => $post]);
Post\Collection::remove($uriid, Post\Collection::FEATURED);
Post\Collection::remove($post['uri-id'], Post\Collection::FEATURED);
Queue::remove($activity);
}
@ -1335,10 +1335,10 @@ class Processor
}
$id = Item::fetchByLink($post['id']);
if (!empty($id)) {
$item = Post::selectFirst(['uri-id', 'featured'], ['id' => $id]);
$item = Post::selectFirst(['uri-id', 'featured', 'author-id'], ['id' => $id]);
if (!empty($item['uri-id'])) {
if (!$item['featured']) {
Post\Collection::add($item['uri-id'], Post\Collection::FEATURED);
Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id']);
Logger::debug('Added featured post', ['uri-id' => $item['uri-id'], 'contact' => $url]);
$new++;
} else {

23
src/Protocol/ActivityPub/Transmitter.php

@ -59,7 +59,6 @@ class Transmitter
{
const CACHEKEY_FEATURED = 'transmitter:getFeatured:';
const CACHEKEY_CONTACTS = 'transmitter:getContacts:';
const CACHEKEY_OUTBOX = 'transmitter:getOutbox:';
/**
* Add relay servers to the list of inboxes
@ -251,14 +250,6 @@ class Transmitter
*/
public static function getOutbox(array $owner, int $page = null, string $requester = '', bool $nocache = false): array
{
if (empty($page)) {
$cachekey = self::CACHEKEY_OUTBOX . $owner['uid'];
$result = DI::cache()->get($cachekey);
if (!$nocache && !is_null($result)) {
return $result;
}
}
$condition = ['private' => [Item::PUBLIC, Item::UNLISTED]];
if (!empty($requester)) {
@ -283,12 +274,12 @@ class Transmitter
'visible' => true
]);
$count = Post::count($condition);
$apcontact = APContact::getByURL($owner['url']);
$data = ['@context' => ActivityPub::CONTEXT];
$data['id'] = DI::baseUrl() . '/outbox/' . $owner['nickname'];
$data['type'] = 'OrderedCollection';
$data['totalItems'] = $count;
$data['totalItems'] = $apcontact['statuses_count'] ?? 0;
if (!empty($page)) {
$data['id'] .= '?' . http_build_query(['page' => $page]);
@ -316,15 +307,17 @@ class Transmitter
$data['next'] = DI::baseUrl() . '/outbox/' . $owner['nickname'] . '?page=' . ($page + 1);
}
// Fix the cached total item count when it is lower than the real count
$total = (($page - 1) * 20) + $data['totalItems'];
if ($total > $data['totalItems']) {
$data['totalItems'] = $total;
}
$data['partOf'] = DI::baseUrl() . '/outbox/' . $owner['nickname'];
$data['orderedItems'] = $list;
}
if (!empty($cachekey)) {
DI::cache()->set($cachekey, $data, Duration::DAY);
}
return $data;
}

7
static/dbstructure.config.php

@ -55,7 +55,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1480);
define('DB_UPDATE_VERSION', 1481);
}
return [
@ -284,7 +284,7 @@ return [
"uid_contact-type" => ["uid", "contact-type"],
"uid_self_contact-type" => ["uid", "self", "contact-type"],
"self_network_uid" => ["self", "network", "uid"],
"gsid" => ["gsid"],
"gsid_uid_failed" => ["gsid", "uid", "failed"],
"uri-id" => ["uri-id"],
]
],
@ -971,6 +971,7 @@ return [
"target-uri-id" => ["target-uri-id"],
"parent-uri-id" => ["parent-uri-id"],
"seen_uid" => ["seen", "uid"],
"uid_type_parent-uri-id_actor-id" => ["uid", "type", "parent-uri-id", "actor-id"],
]
],
"notify" => [
@ -1187,10 +1188,12 @@ return [
"fields" => [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => "0 - Featured"],
"author-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Author of the featured post"],
],
"indexes" => [
"PRIMARY" => ["uri-id", "type"],
"type" => ["type"],
"author-id" => ["author-id"],
]
],
"post-content" => [

4
static/dbview.config.php

@ -702,7 +702,7 @@
"fields" => [
"uri-id" => ["post-collection", "uri-id"],
"type" => ["post-collection", "type"],
"cid" => ["post", "author-id"],
"cid" => ["post-collection", "author-id"],
"received" => ["post", "received"],
"created" => ["post", "created"],
"commented" => ["post-thread", "commented"],
@ -710,7 +710,7 @@
"visible" => ["post", "visible"],
"deleted" => ["post", "deleted"],
"thr-parent-id" => ["post", "thr-parent-id"],
"author-id" => ["post", "author-id"],
"author-id" => ["post-collection", "author-id"],
"gravity" => ["post", "gravity"],
],
"query" => "FROM `post-collection`

10
update.php

@ -1100,9 +1100,9 @@ function update_1451()
function update_1457()
{
$pinned = DBA::select('post-thread-user', ['uri-id'], ['pinned' => true]);
$pinned = DBA::select('post-thread-user', ['uri-id', 'author-id'], ['pinned' => true]);
while ($post = DBA::fetch($pinned)) {
Post\Collection::add($post['uri-id'], Post\Collection::FEATURED);
Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']);
}
DBA::close($pinned);
@ -1115,3 +1115,9 @@ function update_1480()
DBA::update('post', ['deleted' => false], ["`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE NOT `deleted`)"]);
return Update::SUCCESS;
}
function update_1481()
{
DBA::e("UPDATE `post-collection` INNER JOIN `post` ON `post`.`uri-id` = `post-collection`.`uri-id` SET `post-collection`.`author-id` = `post`.`author-id` WHERE `post-collection`.`author-id` IS null");
return Update::SUCCESS;
}

Loading…
Cancel
Save