From f29bd23ea8ed2ae82771ee2c78f7fe82e2b123ed Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Jul 2021 13:47:46 +0000 Subject: [PATCH] New function to convert BBCode for a given ID --- database.sql | 7 ++- doc/database/db_event.md | 47 ++++++++++---------- src/Content/Text/BBCode.php | 31 +++++++++++++ src/Model/Contact.php | 10 ++++- src/Model/Event.php | 5 ++- src/Model/Item.php | 2 +- src/Module/Api/Mastodon/Statuses.php | 2 + src/Module/BaseApi.php | 56 ++++++++++++++++++++++++ src/Object/Api/Mastodon/Status.php | 2 +- src/Protocol/ActivityPub/Transmitter.php | 6 +-- src/Protocol/DFRN.php | 2 +- src/Protocol/Feed.php | 4 +- src/Protocol/OStatus.php | 4 +- src/Worker/ExpirePosts.php | 1 + static/dbstructure.config.php | 4 +- 15 files changed, 145 insertions(+), 38 deletions(-) diff --git a/database.sql b/database.sql index 2121fec979..69ae84696e 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2021.09-dev (Siberian Iris) --- DB_UPDATE_VERSION 1425 +-- DB_UPDATE_VERSION 1426 -- ------------------------------------------ @@ -563,6 +563,7 @@ CREATE TABLE IF NOT EXISTS `event` ( `uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'Owner User id', `cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'contact_id (ID of the contact in contact table)', `uri` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the event uri', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'creation time', `edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'last edit time', `start` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'event start time', @@ -581,8 +582,10 @@ CREATE TABLE IF NOT EXISTS `event` ( PRIMARY KEY(`id`), INDEX `uid_start` (`uid`,`start`), INDEX `cid` (`cid`), + INDEX `uri-id` (`uri-id`), FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE, - FOREIGN KEY (`cid`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE + FOREIGN KEY (`cid`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Events'; -- diff --git a/doc/database/db_event.md b/doc/database/db_event.md index 37048599bb..5550d1a1a0 100644 --- a/doc/database/db_event.md +++ b/doc/database/db_event.md @@ -6,28 +6,29 @@ Events Fields ------ -| Field | Description | Type | Null | Key | Default | Extra | -| --------- | ------------------------------------------------------ | ------------------ | ---- | --- | ------------------- | -------------- | -| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment | -| guid | | varchar(255) | NO | | | | -| uid | Owner User id | mediumint unsigned | NO | | 0 | | -| cid | contact_id (ID of the contact in contact table) | int unsigned | NO | | 0 | | -| uri | | varchar(255) | NO | | | | -| created | creation time | datetime | NO | | 0001-01-01 00:00:00 | | -| edited | last edit time | datetime | NO | | 0001-01-01 00:00:00 | | -| start | event start time | datetime | NO | | 0001-01-01 00:00:00 | | -| finish | event end time | datetime | NO | | 0001-01-01 00:00:00 | | -| summary | short description or title of the event | text | YES | | NULL | | -| desc | event description | text | YES | | NULL | | -| location | event location | text | YES | | NULL | | -| type | event or birthday | varchar(20) | NO | | | | -| nofinish | if event does have no end this is 1 | boolean | NO | | 0 | | -| adjust | adjust to timezone of the recipient (0 or 1) | boolean | NO | | 1 | | -| ignore | 0 or 1 | boolean | NO | | 0 | | -| allow_cid | Access Control - list of allowed contact.id '<19><78>' | mediumtext | YES | | NULL | | -| allow_gid | Access Control - list of allowed groups | mediumtext | YES | | NULL | | -| deny_cid | Access Control - list of denied contact.id | mediumtext | YES | | NULL | | -| deny_gid | Access Control - list of denied groups | mediumtext | YES | | NULL | | +| Field | Description | Type | Null | Key | Default | Extra | +| --------- | ---------------------------------------------------------- | ------------------ | ---- | --- | ------------------- | -------------- | +| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment | +| guid | | varchar(255) | NO | | | | +| uid | Owner User id | mediumint unsigned | NO | | 0 | | +| cid | contact_id (ID of the contact in contact table) | int unsigned | NO | | 0 | | +| uri | | varchar(255) | NO | | | | +| uri-id | Id of the item-uri table entry that contains the event uri | int unsigned | YES | | NULL | | +| created | creation time | datetime | NO | | 0001-01-01 00:00:00 | | +| edited | last edit time | datetime | NO | | 0001-01-01 00:00:00 | | +| start | event start time | datetime | NO | | 0001-01-01 00:00:00 | | +| finish | event end time | datetime | NO | | 0001-01-01 00:00:00 | | +| summary | short description or title of the event | text | YES | | NULL | | +| desc | event description | text | YES | | NULL | | +| location | event location | text | YES | | NULL | | +| type | event or birthday | varchar(20) | NO | | | | +| nofinish | if event does have no end this is 1 | boolean | NO | | 0 | | +| adjust | adjust to timezone of the recipient (0 or 1) | boolean | NO | | 1 | | +| ignore | 0 or 1 | boolean | NO | | 0 | | +| allow_cid | Access Control - list of allowed contact.id '<19><78>' | mediumtext | YES | | NULL | | +| allow_gid | Access Control - list of allowed groups | mediumtext | YES | | NULL | | +| deny_cid | Access Control - list of denied contact.id | mediumtext | YES | | NULL | | +| deny_gid | Access Control - list of denied groups | mediumtext | YES | | NULL | | Indexes ------------ @@ -37,6 +38,7 @@ Indexes | PRIMARY | id | | uid_start | uid, start | | cid | cid | +| uri-id | uri-id | Foreign Keys ------------ @@ -45,5 +47,6 @@ Foreign Keys |-------|--------------|--------------| | uid | [user](help/database/db_user) | uid | | cid | [contact](help/database/db_contact) | id | +| uri-id | [item-uri](help/database/db_item-uri) | id | Return to [database documentation](help/database) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index dbd9a5d9aa..954d630eb2 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1263,6 +1263,37 @@ class BBCode return $bbcode; } + /** + * Converts a BBCode message for a given ID to a HTML message + * + * BBcode 2 HTML was written by WAY2WEB.net + * extended to work with Mistpark/Friendica - Mike Macgirvin + * + * Simple HTML values meaning: + * - 0: Friendica display + * - 1: Unused + * - 2: Used for Windows Phone push, Friendica API + * - 3: Used before converting to Markdown in bb2diaspora.php + * - 4: Used for WordPress, Libertree (before Markdown), pump.io and tumblr + * - 5: Unused + * - 6: Unused + * - 7: Used for dfrn, OStatus + * - 8: Used for Twitter, WP backlink text setting + * - 9: ActivityPub + * + * @param int $uriid + * @param string $text + * @param int $simple_html + * @return string + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + public static function convertForItem(int $uriid, string $text, int $simple_html = self::INTERNAL) + { + $try_oembed = ($simple_html == self::INTERNAL); + + return self::convert($text, $try_oembed, $simple_html, false, $uriid); + } + /** * Converts a BBCode message to HTML message * diff --git a/src/Model/Contact.php b/src/Model/Contact.php index c65837bd97..92f3a48bf5 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -1886,7 +1886,7 @@ class Contact { if (Strings::normaliseLink($new_url) != Strings::normaliseLink($old_url)) { Logger::notice('New URL differs from old URL', ['old' => $old_url, 'new' => $new_url]); - // @todo It is to decide what to do when the URL is changed + return; } if (!DBA::update('contact', $fields, ['id' => $id])) { @@ -2073,6 +2073,14 @@ class Contact return false; } + if (Strings::normaliseLink($ret['url']) != Strings::normaliseLink($contact['url'])) { + $cid = self::getIdForURL($ret['url']); + if (!empty($cid) && ($cid != $id)) { + Logger::notice('URL of contact changed.', ['id' => $id, 'new_id' => $cid, 'old' => $contact['url'], 'new' => $ret['url']]); + return self::updateFromProbeArray($cid, $ret); + } + } + if (isset($ret['hide']) && is_bool($ret['hide'])) { $ret['unsearchable'] = $ret['hide']; } diff --git a/src/Model/Event.php b/src/Model/Event.php index 81e33013fb..7791ac88d3 100644 --- a/src/Model/Event.php +++ b/src/Model/Event.php @@ -273,6 +273,7 @@ class Event $event['cid'] = intval($arr['cid'] ?? 0); $event['guid'] = ($arr['guid'] ?? '') ?: System::createUUID(); $event['uri'] = ($arr['uri'] ?? '') ?: Item::newURI($event['uid'], $event['guid']); + $event['uri-id'] = ItemURI::insert(['uri' => $event['uri'], 'guid' => $event['guid']]); $event['type'] = ($arr['type'] ?? '') ?: 'event'; $event['summary'] = $arr['summary'] ?? ''; $event['desc'] = $arr['desc'] ?? ''; @@ -937,7 +938,7 @@ class Event $tpl = Renderer::getMarkupTemplate('event_stream_item.tpl'); $return = Renderer::replaceMacros($tpl, [ '$id' => $item['event-id'], - '$title' => BBCode::convert($item['event-summary']), + '$title' => BBCode::convertForItem($item['uri-id'], $item['event-summary']), '$dtstart_label' => DI::l10n()->t('Starts:'), '$dtstart_title' => $dtstart_title, '$dtstart_dt' => $dtstart_dt, @@ -955,7 +956,7 @@ class Event '$author_name' => $item['author-name'], '$author_link' => $profile_link, '$author_avatar' => $item['author-avatar'], - '$description' => BBCode::convert($item['event-desc']), + '$description' => BBCode::convertForItem($item['uri-id'], $item['event-desc']), '$location_label' => DI::l10n()->t('Location:'), '$show_map_label' => DI::l10n()->t('Show map'), '$hide_map_label' => DI::l10n()->t('Hide map'), diff --git a/src/Model/Item.php b/src/Model/Item.php index ac43e8a05d..0a4af82bc9 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -2640,7 +2640,7 @@ class Item ) { self::addRedirToImageTags($item); - $item['rendered-html'] = BBCode::convert($item['body'], true, BBCode::INTERNAL, false, $item['uri-id']); + $item['rendered-html'] = BBCode::convertForItem($item['uri-id'], $item['body']); $item['rendered-hash'] = hash('md5', BBCode::VERSION . '::' . $body); $hook_data = ['item' => $item, 'rendered-html' => $item['rendered-html'], 'rendered-hash' => $item['rendered-hash']]; diff --git a/src/Module/Api/Mastodon/Statuses.php b/src/Module/Api/Mastodon/Statuses.php index 77915cab27..1007bea0be 100644 --- a/src/Module/Api/Mastodon/Statuses.php +++ b/src/Module/Api/Mastodon/Statuses.php @@ -140,6 +140,8 @@ class Statuses extends BaseApi $item['gravity'] = GRAVITY_COMMENT; $item['object-type'] = Activity\ObjectType::COMMENT; } else { + self::checkThrottleLimit(); + $item['gravity'] = GRAVITY_PARENT; $item['object-type'] = Activity\ObjectType::NOTE; } diff --git a/src/Module/BaseApi.php b/src/Module/BaseApi.php index f5a16da765..d2aa9662c1 100644 --- a/src/Module/BaseApi.php +++ b/src/Module/BaseApi.php @@ -25,9 +25,11 @@ use Friendica\BaseModule; use Friendica\Core\Logger; use Friendica\Core\System; use Friendica\DI; +use Friendica\Model\Post; use Friendica\Network\HTTPException; use Friendica\Security\BasicAuth; use Friendica\Security\OAuth; +use Friendica\Util\DateTimeFormat; use Friendica\Util\HTTPInputData; require_once __DIR__ . '/../../include/api.php'; @@ -282,6 +284,60 @@ class BaseApi extends BaseModule } } + public static function checkThrottleLimit() + { + $uid = self::getCurrentUserID(); + + // Check for throttling (maximum posts per day, week and month) + $throttle_day = DI::config()->get('system', 'throttle_limit_day'); + if ($throttle_day > 0) { + $datefrom = date(DateTimeFormat::MYSQL, time() - 24*60*60); + + $condition = ["`gravity` = ? AND `uid` = ? AND `wall` AND `received` > ?", GRAVITY_PARENT, $uid, $datefrom]; + $posts_day = Post::count($condition); + + if ($posts_day > $throttle_day) { + Logger::info('Daily posting limit reached', ['uid' => $uid, 'posts' => $posts_day, 'limit' => $throttle_day]); + $error = DI::l10n()->t('Too Many Requests'); + $error_description = DI::l10n()->tt("Daily posting limit of %d post reached. The post was rejected.", "Daily posting limit of %d posts reached. The post was rejected.", $throttle_day); + $errorobj = new \Friendica\Object\Api\Mastodon\Error($error, $error_description); + System::jsonError(429, $errorobj->toArray()); + } + } + + $throttle_week = DI::config()->get('system', 'throttle_limit_week'); + if ($throttle_week > 0) { + $datefrom = date(DateTimeFormat::MYSQL, time() - 24*60*60*7); + + $condition = ["`gravity` = ? AND `uid` = ? AND `wall` AND `received` > ?", GRAVITY_PARENT, $uid, $datefrom]; + $posts_week = Post::count($condition); + + if ($posts_week > $throttle_week) { + Logger::info('Weekly posting limit reached', ['uid' => $uid, 'posts' => $posts_week, 'limit' => $throttle_week]); + $error = DI::l10n()->t('Too Many Requests'); + $error_description = DI::l10n()->tt("Weekly posting limit of %d post reached. The post was rejected.", "Weekly posting limit of %d posts reached. The post was rejected.", $throttle_week); + $errorobj = new \Friendica\Object\Api\Mastodon\Error($error, $error_description); + System::jsonError(429, $errorobj->toArray()); + } + } + + $throttle_month = DI::config()->get('system', 'throttle_limit_month'); + if ($throttle_month > 0) { + $datefrom = date(DateTimeFormat::MYSQL, time() - 24*60*60*30); + + $condition = ["`gravity` = ? AND `uid` = ? AND `wall` AND `received` > ?", GRAVITY_PARENT, $uid, $datefrom]; + $posts_month = Post::count($condition); + + if ($posts_month > $throttle_month) { + Logger::info('Monthly posting limit reached', ['uid' => $uid, 'posts' => $posts_month, 'limit' => $throttle_month]); + $error = DI::l10n()->t('Too Many Requests'); + $error_description = DI::l10n()->t("Monthly posting limit of %d post reached. The post was rejected.", "Monthly posting limit of %d posts reached. The post was rejected.", $throttle_month); + $errorobj = new \Friendica\Object\Api\Mastodon\Error($error, $error_description); + System::jsonError(429, $errorobj->toArray()); + } + } + } + /** * Get user info array. * diff --git a/src/Object/Api/Mastodon/Status.php b/src/Object/Api/Mastodon/Status.php index f476710d4e..2c60d4ee36 100644 --- a/src/Object/Api/Mastodon/Status.php +++ b/src/Object/Api/Mastodon/Status.php @@ -131,7 +131,7 @@ class Status extends BaseDataTransferObject $this->muted = $userAttributes->muted; $this->bookmarked = $userAttributes->bookmarked; $this->pinned = $userAttributes->pinned; - $this->content = BBCode::convert($item['raw-body'] ?? $item['body'], false, BBCode::API, false, $item['uri-id']); + $this->content = BBCode::convertForItem($item['uri-id'], ($item['raw-body'] ?? $item['body']), BBCode::API); $this->reblog = $reblog; $this->application = $application->toArray(); $this->account = $account->toArray(); diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index 652e765294..a60b7c7645 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -1464,7 +1464,7 @@ class Transmitter { $event = []; $event['name'] = $item['event-summary']; - $event['content'] = BBCode::convert($item['event-desc'], false, BBCode::ACTIVITYPUB, false, $item['uri-id']); + $event['content'] = BBCode::convertForItem($item['uri-id'], $item['event-desc'], BBCode::ACTIVITYPUB); $event['startTime'] = DateTimeFormat::utc($item['event-start'] . '+00:00', DateTimeFormat::ATOM); if (!$item['event-nofinish']) { @@ -1571,7 +1571,7 @@ class Transmitter $regexp = "/[@!]\[url\=([^\[\]]*)\].*?\[\/url\]/ism"; $body = preg_replace_callback($regexp, ['self', 'mentionCallback'], $body); - $data['content'] = BBCode::convert($body, false, BBCode::ACTIVITYPUB, false, $item['uri-id']); + $data['content'] = BBCode::convertForItem($item['uri-id'], $body, BBCode::ACTIVITYPUB); } // The regular "content" field does contain a minimized HTML. This is done since systems like @@ -1583,7 +1583,7 @@ class Transmitter $richbody = preg_replace_callback($regexp, ['self', 'mentionCallback'], $item['body']); $richbody = BBCode::removeAttachment($richbody); - $data['contentMap'][$language] = BBCode::convert($richbody, false, BBCode::EXTERNAL, false, $item['uri-id']); + $data['contentMap'][$language] = BBCode::convertForItem($item['uri-id'], $richbody, BBCode::EXTERNAL); } $data['source'] = ['content' => $item['body'], 'mediaType' => "text/bbcode"]; diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 9ac40c0a2d..d8db7251b8 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -918,7 +918,7 @@ class DFRN $htmlbody = "[b]" . $item['title'] . "[/b]\n\n" . $htmlbody; } - $htmlbody = BBCode::convert($htmlbody, false, BBCode::OSTATUS, false, $item['uri-id']); + $htmlbody = BBCode::convert($item['uri-id'], $htmlbody, BBCode::OSTATUS); } $author = self::addEntryAuthor($doc, "author", $item["author-link"], $item); diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index bed67faacd..47248bde1b 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -1109,7 +1109,7 @@ class Feed $body = OStatus::formatPicturePost($item['body'], $item['uri-id']); - $body = BBCode::convert($body, false, BBCode::OSTATUS, false, $item['uri-id']); + $body = BBCode::convertForItem($item['uri-id'], $body, BBCode::OSTATUS, false); XML::addElement($doc, $entry, "content", $body, ["type" => "html"]); @@ -1186,7 +1186,7 @@ class Feed private static function getTitle(array $item) { if ($item['title'] != '') { - return BBCode::convert($item['title'], false, BBCode::OSTATUS, false, $item['uri-id']); + return BBCode::convertForItem($item['uri-id'], $item['title'], BBCode::OSTATUS); } // Fetch information about the post diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index 6318f0940a..8b02eb00e9 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -1803,7 +1803,7 @@ class OStatus if (!$toplevel) { if (!empty($item['title'])) { - $title = BBCode::convert($item['title'], false, BBCode::OSTATUS, false, $item['uri-id']); + $title = BBCode::convertForItem($item['uri-id'], $item['title'], BBCode::OSTATUS); } else { $title = sprintf("New note by %s", $owner["nick"]); } @@ -1892,7 +1892,7 @@ class OStatus $body = "[b]".$item['title']."[/b]\n\n".$body; } - $body = BBCode::convert($body, false, BBCode::OSTATUS, false, $item['uri-id']); + $body = BBCode::convertForItem($item['uri-id'], $body, BBCode::OSTATUS); XML::addElement($doc, $entry, "content", $body, ["type" => "html"]); diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index 3659edfdee..3bdfd0c26f 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -183,6 +183,7 @@ class ExpirePosts AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `mail` WHERE `uri-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `uri-id` FROM `event` WHERE `uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]); diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index c774b85705..f302f1ecc4 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1425); + define('DB_UPDATE_VERSION', 1426); } return [ @@ -628,6 +628,7 @@ return [ "uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "foreign" => ["user" => "uid"], "comment" => "Owner User id"], "cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id"], "comment" => "contact_id (ID of the contact in contact table)"], "uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the event uri"], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "creation time"], "edited" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "last edit time"], "start" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "event start time"], @@ -648,6 +649,7 @@ return [ "PRIMARY" => ["id"], "uid_start" => ["uid", "start"], "cid" => ["cid"], + "uri-id" => ["uri-id"], ] ], "fcontact" => [