From c8954a25ba8a2191509c9079eeb43d8707bb6f95 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 05:24:51 +0000 Subject: [PATCH 01/11] Limit the tag query to a range of one day Fixes #8619 --- mod/network.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mod/network.php b/mod/network.php index 1506c70737..1912ce7881 100644 --- a/mod/network.php +++ b/mod/network.php @@ -786,6 +786,13 @@ function networkThreadedView(App $a, $update, $parent) $top_limit = DateTimeFormat::utcNow(); } + // Handle bad performance situations when the distance between top and bottom is too high + // See issue https://github.com/friendica/friendica/issues/8619 + if (strtotime($top_limit) - strtotime($bottom_limit) > 86400) { + // Set the bottom limit to one day in the past at maximum + $bottom_limit = DateTimeFormat::utc(date('c', strtotime($top_limit) - 86400)); + } + $items = DBA::p("SELECT `item`.`parent-uri` AS `uri`, 0 AS `item_id`, `item`.$ordering AS `order_date`, `author`.`url` AS `author-link` FROM `item` STRAIGHT_JOIN (SELECT `uri-id` FROM `tag-search-view` WHERE `name` IN (SELECT SUBSTR(`term`, 2) FROM `search` WHERE `uid` = ? AND `term` LIKE '#%') AND `uid` = 0) AS `tag-search` From d771c50d636a5cd3467746aa845b71bdd40f1a5f Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 06:13:58 +0000 Subject: [PATCH 02/11] Issue 8565: Sanitize input data Fixes #8565 --- src/Model/GServer.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index bc189af9d6..1eb7ec0e13 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -339,7 +339,7 @@ class GServer * @param string $server_url address of the server * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function discoverRelay(string $server_url) + public static function discoverRelay(string $server_url) { Logger::info('Discover relay data', ['server' => $server_url]); @@ -353,6 +353,15 @@ class GServer return; } + // Sanitize incoming data, see https://github.com/friendica/friendica/issues/8565 + $data['subscribe'] = (bool)$data['subscribe'] ?? false; + + if (!$data['subscribe'] || empty($data['scope']) || !in_array(strtolower($data['scope']), ['all', 'tags'])) { + $data['scope'] = ''; + $data['subscribe'] = false; + $data['tags'] = []; + } + $gserver = DBA::selectFirst('gserver', ['id', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]); if (!DBA::isResult($gserver)) { return; From 4d4678ceb6dcbf9bb055b30bfd39f5f47fcd2156 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 09:03:56 +0000 Subject: [PATCH 03/11] New experimental database configurations --- src/Database/Database.php | 9 +++++---- static/defaults.config.php | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 7adb88ffa8..4b39a13e68 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -135,8 +135,9 @@ class Database } $this->emulate_prepares = (bool)$this->configCache->get('database', 'emulate_prepares'); + $this->pdo_emulate_prepares = (bool)$this->configCache->get('database', 'pdo_emulate_prepares'); - if (class_exists('\PDO') && in_array('mysql', PDO::getAvailableDrivers())) { + if (!$this->configCache->get('database', 'disable_pdo') && class_exists('\PDO') && in_array('mysql', PDO::getAvailableDrivers())) { $this->driver = 'pdo'; $connect = "mysql:host=" . $server . ";dbname=" . $db; @@ -150,7 +151,7 @@ class Database try { $this->connection = @new PDO($connect, $user, $pass); - $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); + $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, $this->pdo_emulate_prepares); $this->connected = true; } catch (PDOException $e) { $this->connected = false; @@ -1029,7 +1030,7 @@ class Database $success = $this->e("LOCK TABLES " . DBA::buildTableString($table) . " WRITE"); if ($this->driver == 'pdo') { - $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); + $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, $this->pdo_emulate_prepares); } if (!$success) { @@ -1062,7 +1063,7 @@ class Database $success = $this->e("UNLOCK TABLES"); if ($this->driver == 'pdo') { - $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); + $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, $this->pdo_emulate_prepares); $this->e("SET autocommit=1"); } else { $this->connection->autocommit(true); diff --git a/static/defaults.config.php b/static/defaults.config.php index d0725aa9bc..4ef006496d 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -45,8 +45,23 @@ return [ 'database' => '', // charset (String) - // Database connexion charset. Changing this value will likely corrupt special characters. + // Database connection charset. Changing this value will likely corrupt special characters. 'charset' => 'utf8mb4', + + // emulate_prepares (Boolean) (Experimental) + // If enabled, prepared statements will be emulated. + // In combination with MySQLi this will cast all return values to strings. + 'emulate_prepares' => false, + + // pdo_emulate_prepares (Boolean) (Experimental) + // If enabled, the builtin emulation for prepared statements is used. + // Due to limitations of that emulation (all return values are casted as strings) + // this will most likely cause issues and should not be used on production systems. + 'pdo_emulate_prepares' => false, + + // disable_pdo (Boolean) + // PDO is used by default (if available). Otherwise MySQLi will be used. + 'disable_pdo' => false, ], 'config' => [ // admin_email (Comma-separated list) From 56e363b84b1b39319248e7e7c6dc25db6040ab0f Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 10:03:11 +0000 Subject: [PATCH 04/11] Fix tests --- tests/Util/Database/StaticDatabase.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Util/Database/StaticDatabase.php b/tests/Util/Database/StaticDatabase.php index f2ed6c700e..c95b690c63 100644 --- a/tests/Util/Database/StaticDatabase.php +++ b/tests/Util/Database/StaticDatabase.php @@ -56,6 +56,8 @@ class StaticDatabase extends Database $this->driver = 'pdo'; $this->connection = self::$staticConnection; $this->connected = true; + $this->emulate_prepares = false; + $this->pdo_emulate_prepares = false; return $this->connected; } From 7ace1049bb965c5029f06e6124c4608a6d0ce95f Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 10:56:19 +0000 Subject: [PATCH 05/11] Issue 8572: Ensure to log database errors --- src/Database/Database.php | 41 +++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 7adb88ffa8..4944529744 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -504,6 +504,7 @@ class Database $sql = "/*" . System::callstack() . " */ " . $sql; } + $is_error = false; $this->error = ''; $this->errorno = 0; $this->affected_rows = 0; @@ -533,6 +534,7 @@ class Database $this->error = $errorInfo[2]; $this->errorno = $errorInfo[1]; $retval = false; + $is_error = true; break; } $this->affected_rows = $retval->rowCount(); @@ -545,6 +547,7 @@ class Database $this->error = $errorInfo[2]; $this->errorno = $errorInfo[1]; $retval = false; + $is_error = true; break; } @@ -562,6 +565,7 @@ class Database $this->error = $errorInfo[2]; $this->errorno = $errorInfo[1]; $retval = false; + $is_error = true; } else { $retval = $stmt; $this->affected_rows = $retval->rowCount(); @@ -580,6 +584,7 @@ class Database $this->error = $this->connection->error; $this->errorno = $this->connection->errno; $retval = false; + $is_error = true; } else { if (isset($retval->num_rows)) { $this->affected_rows = $retval->num_rows; @@ -596,6 +601,7 @@ class Database $this->error = $stmt->error; $this->errorno = $stmt->errno; $retval = false; + $is_error = true; break; } @@ -623,6 +629,7 @@ class Database $this->error = $this->connection->error; $this->errorno = $this->connection->errno; $retval = false; + $is_error = true; } else { $stmt->store_result(); $retval = $stmt; @@ -631,6 +638,16 @@ class Database break; } + // See issue https://github.com/friendica/friendica/issues/8572 + // Ensure that we always get an error message on an error. + if ($is_error && empty($this->errorno)) { + $this->errorno = -1; + } + + if ($is_error && empty($this->error)) { + $this->error = 'Unknown database error'; + } + // We are having an own error logging in the function "e" if (($this->errorno != 0) && !$called_from_e) { // We have to preserve the error code, somewhere in the logging it get lost @@ -642,8 +659,8 @@ class Database } $this->logger->error('DB Error', [ - 'code' => $this->errorno, - 'error' => $this->error, + 'code' => $errorno, + 'error' => $error, 'callstack' => System::callstack(8), 'params' => $this->replaceParameters($sql, $args), ]); @@ -654,21 +671,21 @@ class Database // It doesn't make sense to continue when the database connection was lost if ($this->in_retrial) { $this->logger->notice('Giving up retrial because of database error', [ - 'code' => $this->errorno, - 'error' => $this->error, + 'code' => $errorno, + 'error' => $error, ]); } else { $this->logger->notice('Couldn\'t reconnect after database error', [ - 'code' => $this->errorno, - 'error' => $this->error, + 'code' => $errorno, + 'error' => $error, ]); } exit(1); } else { // We try it again $this->logger->notice('Reconnected after database error', [ - 'code' => $this->errorno, - 'error' => $this->error, + 'code' => $errorno, + 'error' => $error, ]); $this->in_retrial = true; $ret = $this->p($sql, $args); @@ -745,8 +762,8 @@ class Database } $this->logger->error('DB Error', [ - 'code' => $this->errorno, - 'error' => $this->error, + 'code' => $errorno, + 'error' => $error, 'callstack' => System::callstack(8), 'params' => $this->replaceParameters($sql, $params), ]); @@ -755,8 +772,8 @@ class Database // A reconnect like in $this->p could be dangerous with modifications if ($errorno == 2006) { $this->logger->notice('Giving up because of database error', [ - 'code' => $this->errorno, - 'error' => $this->error, + 'code' => $errorno, + 'error' => $error, ]); exit(1); } From bc26c980f0b0143cea59a4ff013805141bb20ed4 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 13:59:05 +0000 Subject: [PATCH 06/11] Reverts test changes --- src/Model/GServer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 1eb7ec0e13..f750ed99ea 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -339,7 +339,7 @@ class GServer * @param string $server_url address of the server * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function discoverRelay(string $server_url) + private static function discoverRelay(string $server_url) { Logger::info('Discover relay data', ['server' => $server_url]); From 3b1d89252a8a9d4e5ff74ceec44ab2b6dc844dc6 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 15:01:27 +0000 Subject: [PATCH 07/11] Issue 8582: "cid" in "network" is now "contactid" Fixes #8582 --- include/conversation.php | 22 +++++++++++----------- mod/network.php | 18 +++++++++--------- src/Content/ForumManager.php | 2 +- view/theme/vier/theme.php | 2 +- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/include/conversation.php b/include/conversation.php index 5b7ccbd8a1..0eb94181f0 100644 --- a/include/conversation.php +++ b/include/conversation.php @@ -376,17 +376,17 @@ function conversation(App $a, array $items, $mode, $update, $preview = false, $o . "\r\n"; } diff --git a/mod/network.php b/mod/network.php index 1506c70737..1ef7c67f4c 100644 --- a/mod/network.php +++ b/mod/network.php @@ -58,8 +58,8 @@ function network_init(App $a) $group_id = (($a->argc > 1 && is_numeric($a->argv[1])) ? intval($a->argv[1]) : 0); $cid = 0; - if (!empty($_GET['cid'])) { - $cid = $_GET['cid']; + if (!empty($_GET['contactid'])) { + $cid = $_GET['contactid']; $_GET['nets'] = ''; $group_id = 0; } @@ -466,12 +466,12 @@ function networkThreadedView(App $a, $update, $parent) $o = ''; - $cid = intval($_GET['cid'] ?? 0); - $star = intval($_GET['star'] ?? 0); - $bmark = intval($_GET['bmark'] ?? 0); - $conv = intval($_GET['conv'] ?? 0); + $cid = intval($_GET['contactid'] ?? 0); + $star = intval($_GET['star'] ?? 0); + $bmark = intval($_GET['bmark'] ?? 0); + $conv = intval($_GET['conv'] ?? 0); $order = Strings::escapeTags(($_GET['order'] ?? '') ?: 'activity'); - $nets = $_GET['nets'] ?? ''; + $nets = $_GET['nets'] ?? ''; $allowedCids = []; if ($cid) { @@ -891,8 +891,8 @@ function network_tabs(App $a) $cmd = DI::args()->getCommand(); $def_param = []; - if (!empty($_GET['cid'])) { - $def_param['cid'] = $_GET['cid']; + if (!empty($_GET['contactid'])) { + $def_param['contactid'] = $_GET['contactid']; } // tabs diff --git a/src/Content/ForumManager.php b/src/Content/ForumManager.php index 7d3cb89a71..9441e9dbb0 100644 --- a/src/Content/ForumManager.php +++ b/src/Content/ForumManager.php @@ -126,7 +126,7 @@ class ForumManager $selected = (($cid == $contact['id']) ? ' forum-selected' : ''); $entry = [ - 'url' => 'network?cid=' . $contact['id'], + 'url' => 'network?contactid=' . $contact['id'], 'external_url' => Contact::magicLink($contact['url']), 'name' => $contact['name'], 'cid' => $contact['id'], diff --git a/view/theme/vier/theme.php b/view/theme/vier/theme.php index cc3a668e6a..cec5e5ba63 100644 --- a/view/theme/vier/theme.php +++ b/view/theme/vier/theme.php @@ -202,7 +202,7 @@ function vier_community_info() $selected = (($cid == $contact['id']) ? ' forum-selected' : ''); $entry = [ - 'url' => 'network?cid=' . $contact['id'], + 'url' => 'network?contactid=' . $contact['id'], 'external_url' => Contact::magicLink($contact['url']), 'name' => $contact['name'], 'cid' => $contact['id'], From d7f0ffdbc17df9633cb008fffd39afb0f7416d20 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 16:55:54 +0000 Subject: [PATCH 08/11] Issue 8458: Display big emojis Fixes #8458 --- src/Content/Text/BBCode.php | 10 +++++++++- static/defaults.config.php | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index 98c1552930..508a325ca1 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1724,7 +1724,15 @@ class BBCode // Replace non graphical smilies for external posts if (!$nosmile && !$for_plaintext) { - $text = Smilies::replace($text); + $oldtext = $text; + $text = Smilies::replace($text); + if (DI::config()->get('system', 'big_emojis') && ($simple_html != self::DIASPORA) && ($oldtext != $text)) { + $conv = html_entity_decode(str_replace([' ', "\n", "\r"], '', $text)); + // Emojis are always 4 byte Unicode characters + if (strlen($conv) / mb_strlen($conv) == 4) { + $text = '' . $text . ''; + } + } } if (!$for_plaintext) { diff --git a/static/defaults.config.php b/static/defaults.config.php index 4ef006496d..a1d50bb84f 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -103,6 +103,10 @@ return [ // chose "Remember me" when logging in is considered logged out. 'auth_cookie_lifetime' => 7, + // big_emojis (Boolean) + // Display "Emoji Only" posts in big. + 'big_emojis' => false, + // block_local_dir (Boolean) // Deny public access to the local user directory. 'block_local_dir' => false, From 23046425350d8ef48ecb5d253903e34f9651a0d5 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 17:28:40 +0000 Subject: [PATCH 09/11] Fix test --- tests/src/Content/Text/BBCodeTest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/src/Content/Text/BBCodeTest.php b/tests/src/Content/Text/BBCodeTest.php index 1a1d06dc7b..35dff87d9f 100644 --- a/tests/src/Content/Text/BBCodeTest.php +++ b/tests/src/Content/Text/BBCodeTest.php @@ -58,6 +58,9 @@ class BBCodeTest extends MockedTest $this->configMock->shouldReceive('get') ->with('system', 'no_smilies') ->andReturn(false); + $this->configMock->shouldReceive('get') + ->with('system', 'big_emojis') + ->andReturn(false); $l10nMock = \Mockery::mock(L10n::class); $l10nMock->shouldReceive('t')->withAnyArgs()->andReturnUsing(function ($args) { return $args; }); From e737eea17ddc624ec8229db2a620692de396a02f Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 17 May 2020 21:46:54 +0000 Subject: [PATCH 10/11] Issue 8458-2: Now all unicode emojis should work --- src/Content/Text/BBCode.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index 508a325ca1..258f52523c 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1724,14 +1724,14 @@ class BBCode // Replace non graphical smilies for external posts if (!$nosmile && !$for_plaintext) { - $oldtext = $text; $text = Smilies::replace($text); - if (DI::config()->get('system', 'big_emojis') && ($simple_html != self::DIASPORA) && ($oldtext != $text)) { - $conv = html_entity_decode(str_replace([' ', "\n", "\r"], '', $text)); - // Emojis are always 4 byte Unicode characters - if (strlen($conv) / mb_strlen($conv) == 4) { - $text = '' . $text . ''; - } + } + + if (DI::config()->get('system', 'big_emojis') && ($simple_html != self::DIASPORA)) { + $conv = html_entity_decode(str_replace([' ', "\n", "\r"], '', $text)); + // Emojis are always 4 byte Unicode characters + if (strlen($conv) / mb_strlen($conv) == 4) { + $text = '' . $text . ''; } } From efb1c630fdb8ee191eb5316f0853b8fccaff09c3 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 18 May 2020 02:58:08 +0000 Subject: [PATCH 11/11] Fix warnings ("Divide by zero" and "key parameter is not a valid public key") --- src/Content/Text/BBCode.php | 4 ++-- src/Protocol/Diaspora.php | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index 258f52523c..67dbe3b19a 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1727,10 +1727,10 @@ class BBCode $text = Smilies::replace($text); } - if (DI::config()->get('system', 'big_emojis') && ($simple_html != self::DIASPORA)) { + if (!$for_plaintext && DI::config()->get('system', 'big_emojis') && ($simple_html != self::DIASPORA)) { $conv = html_entity_decode(str_replace([' ', "\n", "\r"], '', $text)); // Emojis are always 4 byte Unicode characters - if (strlen($conv) / mb_strlen($conv) == 4) { + if (!empty($conv) && (strlen($conv) / mb_strlen($conv) == 4)) { $text = '' . $text . ''; } } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index dd1f678d58..243550862b 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -3139,7 +3139,9 @@ class Diaspora $json = json_encode(["iv" => $b_iv, "key" => $b_aes_key]); $encrypted_key_bundle = ""; - openssl_public_encrypt($json, $encrypted_key_bundle, $pubkey); + if (!@openssl_public_encrypt($json, $encrypted_key_bundle, $pubkey)) { + return false; + } $json_object = json_encode( ["aes_key" => base64_encode($encrypted_key_bundle),