Merge pull request #10396 from annando/public-timeline

API: improved speed of the public timeline

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2021.06-rc (Siberian Iris)
--- DB_UPDATE_VERSION 1421
+-- DB_UPDATE_VERSION 1422
 -- ------------------------------------------
 
 
@@ -1874,7 +1874,9 @@ CREATE VIEW `post-thread-user-view` AS SELECT
 	`parent-post`.`author-id` AS `parent-author-id`,
 	`parent-post-author`.`url` AS `parent-author-link`,
 	`parent-post-author`.`name` AS `parent-author-name`,
-	`parent-post-author`.`network` AS `parent-author-network`
+	`parent-post-author`.`network` AS `parent-author-network`,
+	`parent-post-author`.`blocked` AS `parent-author-blocked`,
+	`parent-post-author`.`hidden` AS `parent-author-hidden`
 	FROM `post-thread-user`
 			INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
 			STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
@@ -1938,6 +1940,31 @@ CREATE VIEW `post-view` AS SELECT
 	`post-content`.`target-type` AS `target-type`,
 	`post-content`.`target` AS `target`,
 	`post-content`.`resource-id` AS `resource-id`,
+	`post`.`author-id` AS `contact-id`,
+	`author`.`url` AS `contact-link`,
+	`author`.`addr` AS `contact-addr`,
+	`author`.`name` AS `contact-name`,
+	`author`.`nick` AS `contact-nick`,
+	`author`.`thumb` AS `contact-avatar`,
+	`author`.`network` AS `contact-network`,
+	`author`.`blocked` AS `contact-blocked`,
+	`author`.`hidden` AS `contact-hidden`,
+	`author`.`readonly` AS `contact-readonly`,
+	`author`.`archive` AS `contact-archive`,
+	`author`.`pending` AS `contact-pending`,
+	`author`.`rel` AS `contact-rel`,
+	`author`.`uid` AS `contact-uid`,
+	`author`.`contact-type` AS `contact-contact-type`,
+	IF (`post`.`network` IN ('apub', 'dfrn', 'dspr', 'stat'), true, `author`.`writable`) AS `writable`,
+	false AS `self`,
+	`author`.`id` AS `cid`,
+	`author`.`alias` AS `alias`,
+	`author`.`photo` AS `photo`,
+	`author`.`name-date` AS `name-date`,
+	`author`.`uri-date` AS `uri-date`,
+	`author`.`avatar-date` AS `avatar-date`,
+	`author`.`thumb` AS `thumb`,
+	`author`.`dfrn-id` AS `dfrn-id`,
 	`post`.`author-id` AS `author-id`,
 	`author`.`url` AS `author-link`,
 	`author`.`addr` AS `author-addr`,
@@ -1973,7 +2000,9 @@ CREATE VIEW `post-view` AS SELECT
 	`parent-post`.`author-id` AS `parent-author-id`,
 	`parent-post-author`.`url` AS `parent-author-link`,
 	`parent-post-author`.`name` AS `parent-author-name`,
-	`parent-post-author`.`network` AS `parent-author-network`
+	`parent-post-author`.`network` AS `parent-author-network`,
+	`parent-post-author`.`blocked` AS `parent-author-blocked`,
+	`parent-post-author`.`hidden` AS `parent-author-hidden`
 	FROM `post`
 			STRAIGHT_JOIN `post-thread` ON `post-thread`.`uri-id` = `post`.`parent-uri-id`
 			STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = `post`.`author-id`
@@ -2033,6 +2062,31 @@ CREATE VIEW `post-thread-view` AS SELECT
 	`post-content`.`target-type` AS `target-type`,
 	`post-content`.`target` AS `target`,
 	`post-content`.`resource-id` AS `resource-id`,
+	`post-thread`.`author-id` AS `contact-id`,
+	`author`.`url` AS `contact-link`,
+	`author`.`addr` AS `contact-addr`,
+	`author`.`name` AS `contact-name`,
+	`author`.`nick` AS `contact-nick`,
+	`author`.`thumb` AS `contact-avatar`,
+	`author`.`network` AS `contact-network`,
+	`author`.`blocked` AS `contact-blocked`,
+	`author`.`hidden` AS `contact-hidden`,
+	`author`.`readonly` AS `contact-readonly`,
+	`author`.`archive` AS `contact-archive`,
+	`author`.`pending` AS `contact-pending`,
+	`author`.`rel` AS `contact-rel`,
+	`author`.`uid` AS `contact-uid`,
+	`author`.`contact-type` AS `contact-contact-type`,
+	IF (`post`.`network` IN ('apub', 'dfrn', 'dspr', 'stat'), true, `author`.`writable`) AS `writable`,
+	false AS `self`,
+	`author`.`id` AS `cid`,
+	`author`.`alias` AS `alias`,
+	`author`.`photo` AS `photo`,
+	`author`.`name-date` AS `name-date`,
+	`author`.`uri-date` AS `uri-date`,
+	`author`.`avatar-date` AS `avatar-date`,
+	`author`.`thumb` AS `thumb`,
+	`author`.`dfrn-id` AS `dfrn-id`,
 	`post-thread`.`author-id` AS `author-id`,
 	`author`.`url` AS `author-link`,
 	`author`.`addr` AS `author-addr`,
@@ -2068,7 +2122,9 @@ CREATE VIEW `post-thread-view` AS SELECT
 	`parent-post`.`author-id` AS `parent-author-id`,
 	`parent-post-author`.`url` AS `parent-author-link`,
 	`parent-post-author`.`name` AS `parent-author-name`,
-	`parent-post-author`.`network` AS `parent-author-network`
+	`parent-post-author`.`network` AS `parent-author-network`,
+	`parent-post-author`.`blocked` AS `parent-author-blocked`,
+	`parent-post-author`.`hidden` AS `parent-author-hidden`
 	FROM `post-thread`
 			INNER JOIN `post` ON `post`.`uri-id` = `post-thread`.`uri-id`
 			STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = `post-thread`.`author-id`

src/Factory/Api/Mastodon/Status.php

@@ -72,9 +72,9 @@ class Status extends BaseFactory
 		$account = DI::mstdnAccount()->createFromContactId($item['author-id']);
 
 		$counts = new \Friendica\Object\Api\Mastodon\Status\Counts(
-			Post::count(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_COMMENT, 'deleted' => false], [], false),
+			Post::countPosts(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_COMMENT, 'deleted' => false], []),
-			Post::count(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_ACTIVITY, 'vid' => Verb::getID(Activity::ANNOUNCE), 'deleted' => false], [], false),
+			Post::countPosts(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_ACTIVITY, 'vid' => Verb::getID(Activity::ANNOUNCE), 'deleted' => false], []),
-			Post::count(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_ACTIVITY, 'vid' => Verb::getID(Activity::LIKE), 'deleted' => false], [], false)
+			Post::countPosts(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_ACTIVITY, 'vid' => Verb::getID(Activity::LIKE), 'deleted' => false], [])
 		);
 
 		$userAttributes = new \Friendica\Object\Api\Mastodon\Status\UserAttributes(

src/Model/Post.php

@@ -124,24 +124,22 @@ class Post
 	}
 
 	/**
-	 * Check if post data exists
+	 * Check if post-user-view records exists
 	 *
 	 * @param array $condition array of fields for condition
-	 * @param bool  $user_mode true = post-user-view, false = post-view
 	 *
 	 * @return boolean Are there rows for that condition?
 	 * @throws \Exception
 	 */
-	public static function exists($condition, bool $user_mode = true) {
+	public static function exists($condition) {
-		return DBA::exists($user_mode ? 'post-user-view' : 'post-view', $condition);
+		return DBA::exists('post-user-view', $condition);
 	}
 
 	/**
-	 * Counts the posts satisfying the provided condition
+	 * Counts the post-user-view records satisfying the provided condition
 	 *
 	 * @param array        $condition array of fields for condition
 	 * @param array        $params    Array of several parameters
-	 * @param bool         $user_mode true = post-user-view, false = post-view
 	 *
 	 * @return int
 	 *
@@ -153,13 +151,34 @@ class Post
 	 * $count = Post::count($condition);
 	 * @throws \Exception
 	 */
-	public static function count(array $condition = [], array $params = [], bool $user_mode = true)
+	public static function count(array $condition = [], array $params = [])
 	{
-		return DBA::count($user_mode ? 'post-user-view' : 'post-view', $condition, $params);
+		return DBA::count('post-user-view', $condition, $params);
 	}
 
 	/**
-	 * Retrieve a single record from the post table and returns it in an associative array
+	 * Counts the post-view records satisfying the provided condition
+	 *
+	 * @param array        $condition array of fields for condition
+	 * @param array        $params    Array of several parameters
+	 *
+	 * @return int
+	 *
+	 * Example:
+	 * $condition = ["network" => 'dspr'];
+	 * or:
+	 * $condition = ["`network` IN (?, ?)", 1, 'dfrn', 'dspr'];
+	 *
+	 * $count = Post::count($condition);
+	 * @throws \Exception
+	 */
+	public static function countPosts(array $condition = [], array $params = [])
+	{
+		return DBA::count('post-view', $condition, $params);
+	}
+
+	/**
+	 * Retrieve a single record from the post-user-view view and returns it in an associative array
 	 *
 	 * @param array $fields
 	 * @param array $condition
@@ -169,11 +188,11 @@ class Post
 	 * @throws \Exception
 	 * @see   DBA::select
 	 */
-	public static function selectFirst(array $fields = [], array $condition = [], $params = [], bool $user_mode = true)
+	public static function selectFirst(array $fields = [], array $condition = [], $params = [])
 	{
 		$params['limit'] = 1;
 
-		$result = self::select($fields, $condition, $params, $user_mode);
+		$result = self::select($fields, $condition, $params);
 
 		if (is_bool($result)) {
 			return $result;
@@ -185,7 +204,32 @@ class Post
 	}
 
 	/**
-	 * Retrieve a single record from the post-thread table and returns it in an associative array
+	 * Retrieve a single record from the post-view view and returns it in an associative array
+	 *
+	 * @param array $fields
+	 * @param array $condition
+	 * @param array $params
+	 * @return bool|array
+	 * @throws \Exception
+	 * @see   DBA::select
+	 */
+	public static function selectFirstPost(array $fields = [], array $condition = [], $params = [])
+	{
+		$params['limit'] = 1;
+
+		$result = self::selectPosts($fields, $condition, $params);
+
+		if (is_bool($result)) {
+			return $result;
+		} else {
+			$row = self::fetch($result);
+			DBA::close($result);
+			return $row;
+		}
+	}
+
+	/**
+	 * Retrieve a single record from the post-thread-user-view view and returns it in an associative array
 	 *
 	 * @param array $fields
 	 * @param array $condition
@@ -210,7 +254,7 @@ class Post
 	}
 
 	/**
-	 * Select rows from the post table and returns them as an array
+	 * Select rows from the post-user-view view and returns them as an array
 	 *
 	 * @param array $selected  Array of selected fields, empty for all
 	 * @param array $condition Array of fields for condition
@@ -263,23 +307,37 @@ class Post
 	}
 
 	/**
-	 * Select rows from the post table
+	 * Select rows from the post-user-view view
 	 *
 	 * @param array $selected  Array of selected fields, empty for all
 	 * @param array $condition Array of fields for condition
 	 * @param array $params    Array of several parameters
-	 * @param bool  $user_mode true = post-user-view, false = post-view
 	 *
 	 * @return boolean|object
 	 * @throws \Exception
 	 */
-	public static function select(array $selected = [], array $condition = [], $params = [], bool $user_mode = true)
+	public static function select(array $selected = [], array $condition = [], $params = [])
 	{
-		return self::selectView($user_mode ? 'post-user-view' : 'post-view', $selected, $condition, $params);
+		return self::selectView('post-user-view', $selected, $condition, $params);
 	}
 
 	/**
-	 * Select rows from the post table
+	 * Select rows from the post-view view
+	 *
+	 * @param array $selected  Array of selected fields, empty for all
+	 * @param array $condition Array of fields for condition
+	 * @param array $params    Array of several parameters
+	 *
+	 * @return boolean|object
+	 * @throws \Exception
+	 */
+	public static function selectPosts(array $selected = [], array $condition = [], $params = [])
+	{
+		return self::selectView('post-view', $selected, $condition, $params);
+	}
+
+	/**
+	 * Select rows from the post-thread-user-view view
 	 *
 	 * @param array $selected  Array of selected fields, empty for all
 	 * @param array $condition Array of fields for condition
@@ -336,7 +394,7 @@ class Post
 	}
 
 	/**
-	 * Select rows from the post view for a given user
+	 * Select rows from the post-user-view view for a given user
 	 *
 	 * @param integer $uid       User ID
 	 * @param array   $selected  Array of selected fields, empty for all
@@ -351,8 +409,24 @@ class Post
 		return self::selectViewForUser('post-user-view', $uid, $selected, $condition, $params);
 	}
 
-		/**
+	/**
-	 * Select rows from the post view for a given user
+	 * Select rows from the post-view view for a given user
+	 *
+	 * @param integer $uid       User ID
+	 * @param array   $selected  Array of selected fields, empty for all
+	 * @param array   $condition Array of fields for condition
+	 * @param array   $params    Array of several parameters
+	 *
+	 * @return boolean|object
+	 * @throws \Exception
+	 */
+	public static function selectPostsForUser($uid, array $selected = [], array $condition = [], $params = [])
+	{
+		return self::selectViewForUser('post-view', $uid, $selected, $condition, $params);
+	}
+
+	/**
+	 * Select rows from the post-thread-user-view view for a given user
 	 *
 	 * @param integer $uid       User ID
 	 * @param array   $selected  Array of selected fields, empty for all
@@ -368,7 +442,7 @@ class Post
 	}
 
 	/**
-	 * Retrieve a single record from the post view for a given user and returns it in an associative array
+	 * Retrieve a single record from the post-user-view view for a given user and returns it in an associative array
 	 *
 	 * @param integer $uid User ID
 	 * @param array   $selected
@@ -394,7 +468,7 @@ class Post
 	}
 
 	/**
-	 * Select pinned rows from the item table for a given user
+	 * Select pinned rows from the post-thread-user table for a given user
 	 *
 	 * @param integer $uid       User ID
 	 * @param array   $selected  Array of selected fields, empty for all

src/Module/Api/Mastodon/Preferences.php

@@ -52,7 +52,7 @@ class Preferences extends BaseApi
 		$sensitive = false;
 		$language  = $user['language'];
 		$media     = DI::pConfig()->get($uid, 'nsfw', 'disable') ? 'show_all' : 'default';
-		$spoilers  = DI::pConfig()->get($uid, 'system', 'disable_cw');
+		$spoilers  = (bool)DI::pConfig()->get($uid, 'system', 'disable_cw');
 
 		$preferences = new \Friendica\Object\Api\Mastodon\Preferences($visibility, $sensitive, $language, $media, $spoilers);
 

src/Module/Api/Mastodon/Statuses/FavouritedBy.php

@@ -49,7 +49,7 @@ class FavouritedBy extends BaseApi
 			DI::mstdnError()->RecordNotFound();
 		}
 
-		$activities = Post::select(['author-id'], ['thr-parent-id' => $id, 'gravity' => GRAVITY_ACTIVITY, 'verb' => Activity::LIKE], [], false);
+		$activities = Post::selectPosts(['author-id'], ['thr-parent-id' => $id, 'gravity' => GRAVITY_ACTIVITY, 'verb' => Activity::LIKE]);
 
 		$accounts = [];
 

src/Module/Api/Mastodon/Statuses/RebloggedBy.php

@@ -49,7 +49,7 @@ class RebloggedBy extends BaseApi
 			DI::mstdnError()->RecordNotFound();
 		}
 
-		$activities = Post::select(['author-id'], ['thr-parent-id' => $id, 'gravity' => GRAVITY_ACTIVITY, 'verb' => Activity::ANNOUNCE], [], false);
+		$activities = Post::selectPosts(['author-id'], ['thr-parent-id' => $id, 'gravity' => GRAVITY_ACTIVITY, 'verb' => Activity::ANNOUNCE]);
 
 		$accounts = [];
 

src/Module/Api/Mastodon/Timelines/PublicTimeline.php

@@ -58,7 +58,7 @@ class PublicTimeline extends BaseApi
 		$params = ['order' => ['uri-id' => true], 'limit' => $request['limit']];
 
 		$condition = ['gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT], 'private' => Item::PUBLIC,
-			'uid' => 0, 'network' => Protocol::FEDERATED, 'parent-author-blocked' => false, 'parent-author-hidden' => false];
+			'network' => Protocol::FEDERATED, 'parent-author-blocked' => false, 'parent-author-hidden' => false];
 
 		if ($request['local']) {
 			$condition = DBA::mergeConditions($condition, ["`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `origin`)"]);
@@ -95,12 +95,12 @@ class PublicTimeline extends BaseApi
 				["NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `parent-author-id` AND (`blocked` OR `ignored`))", $uid]);
 		}
 
-		$items = Post::selectForUser($uid, ['uri-id', 'uid'], $condition, $params);
+		$items = Post::selectPostsForUser($uid, ['uri-id'], $condition, $params);
 
 		$statuses = [];
 		while ($item = Post::fetch($items)) {
 			self::setBoundaries($item['uri-id']);
-			$statuses[] = DI::mstdnStatus()->createFromUriId($item['uri-id'], $item['uid']);
+			$statuses[] = DI::mstdnStatus()->createFromUriId($item['uri-id'], $uid);
 		}
 		DBA::close($items);
 

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1421);
+	define('DB_UPDATE_VERSION', 1422);
 }
 
 return [

static/dbview.config.php

@@ -356,6 +356,8 @@
 			"parent-author-link" => ["parent-post-author", "url"],
 			"parent-author-name" => ["parent-post-author", "name"],
 			"parent-author-network" => ["parent-post-author", "network"],
+			"parent-author-blocked" => ["parent-post-author", "blocked"],
+			"parent-author-hidden" => ["parent-post-author", "hidden"],
 		],
 		"query" => "FROM `post-thread-user`
 			INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
@@ -417,6 +419,31 @@
 			"target-type" => ["post-content", "target-type"],
 			"target" => ["post-content", "target"],
 			"resource-id" => ["post-content", "resource-id"],
+			"contact-id" => ["post", "author-id"],
+			"contact-link" => ["author", "url"],
+			"contact-addr" => ["author", "addr"],
+			"contact-name" => ["author", "name"],
+			"contact-nick" => ["author", "nick"],
+			"contact-avatar" => ["author", "thumb"],
+			"contact-network" => ["author", "network"],
+			"contact-blocked" => ["author", "blocked"],
+			"contact-hidden" => ["author", "hidden"],
+			"contact-readonly" => ["author", "readonly"],
+			"contact-archive" => ["author", "archive"],
+			"contact-pending" => ["author", "pending"],
+			"contact-rel" => ["author", "rel"],
+			"contact-uid" => ["author", "uid"],
+			"contact-contact-type" => ["author", "contact-type"],
+			"writable" => "IF (`post`.`network` IN ('apub', 'dfrn', 'dspr', 'stat'), true, `author`.`writable`)",
+			"self" => "false",
+			"cid" => ["author", "id"],
+			"alias" => ["author", "alias"],
+			"photo" => ["author", "photo"],
+			"name-date" => ["author", "name-date"],
+			"uri-date" => ["author", "uri-date"],
+			"avatar-date" => ["author", "avatar-date"],
+			"thumb" => ["author", "thumb"],
+			"dfrn-id" => ["author", "dfrn-id"],
 			"author-id" => ["post", "author-id"],
 			"author-link" => ["author", "url"],
 			"author-addr" => ["author", "addr"],
@@ -453,6 +480,8 @@
 			"parent-author-link" => ["parent-post-author", "url"],
 			"parent-author-name" => ["parent-post-author", "name"],
 			"parent-author-network" => ["parent-post-author", "network"],
+			"parent-author-blocked" => ["parent-post-author", "blocked"],
+			"parent-author-hidden" => ["parent-post-author", "hidden"],
 		],
 		"query" => "FROM `post`
 			STRAIGHT_JOIN `post-thread` ON `post-thread`.`uri-id` = `post`.`parent-uri-id`
@@ -510,6 +539,31 @@
 			"target-type" => ["post-content", "target-type"],
 			"target" => ["post-content", "target"],
 			"resource-id" => ["post-content", "resource-id"],
+			"contact-id" => ["post-thread", "author-id"],
+			"contact-link" => ["author", "url"],
+			"contact-addr" => ["author", "addr"],
+			"contact-name" => ["author", "name"],
+			"contact-nick" => ["author", "nick"],
+			"contact-avatar" => ["author", "thumb"],
+			"contact-network" => ["author", "network"],
+			"contact-blocked" => ["author", "blocked"],
+			"contact-hidden" => ["author", "hidden"],
+			"contact-readonly" => ["author", "readonly"],
+			"contact-archive" => ["author", "archive"],
+			"contact-pending" => ["author", "pending"],
+			"contact-rel" => ["author", "rel"],
+			"contact-uid" => ["author", "uid"],
+			"contact-contact-type" => ["author", "contact-type"],
+			"writable" => "IF (`post`.`network` IN ('apub', 'dfrn', 'dspr', 'stat'), true, `author`.`writable`)",
+			"self" => "false",
+			"cid" => ["author", "id"],
+			"alias" => ["author", "alias"],
+			"photo" => ["author", "photo"],
+			"name-date" => ["author", "name-date"],
+			"uri-date" => ["author", "uri-date"],
+			"avatar-date" => ["author", "avatar-date"],
+			"thumb" => ["author", "thumb"],
+			"dfrn-id" => ["author", "dfrn-id"],
 			"author-id" => ["post-thread", "author-id"],
 			"author-link" => ["author", "url"],
 			"author-addr" => ["author", "addr"],
@@ -546,6 +600,8 @@
 			"parent-author-link" => ["parent-post-author", "url"],
 			"parent-author-name" => ["parent-post-author", "name"],
 			"parent-author-network" => ["parent-post-author", "network"],
+			"parent-author-blocked" => ["parent-post-author", "blocked"],
+			"parent-author-hidden" => ["parent-post-author", "hidden"],
 		],
 		"query" => "FROM `post-thread`
 			INNER JOIN `post` ON `post`.`uri-id` = `post-thread`.`uri-id`